02:10:38.561 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
02:10:38.562 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskOg0PFu/test-tmp
02:10:38.562 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
02:10:38.562 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskOg0PFu/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
02:10:38.564 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster2513987901329325466"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
02:10:38.896 [DEBUG - main] (MiniKuduCluster.java:195) Response:
02:10:38.897 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
02:10:38.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:38.900079 30972 mini_oidc.cc:150] Starting JWKS server
02:10:38.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:38.903400 30972 webserver.cc:492] Webserver started at http://127.0.0.1:33017/ using document root <none> and password file <none>
02:10:38.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:38.903856 30972 mini_oidc.cc:186] Starting OIDC Discovery server
02:10:38.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:38.905611 30972 webserver.cc:492] Webserver started at http://127.0.0.1:40681/ using document root <none> and password file <none>
02:10:38.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-10-24T02:10:38Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
02:10:38.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-10-24T02:10:38Z Disabled control of system clock
02:10:38.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:38.969398 30972 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskOg0PFu/build/tsan/bin/kudu
02:10:38.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskOg0PFu/build/tsan/bin/kudu
02:10:38.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster2513987901329325466/master-0/wal
02:10:38.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster2513987901329325466/master-0/data
02:10:38.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:10:38.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:10:38.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:10:38.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:10:38.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:10:38.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:10:38.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster2513987901329325466/master-0/logs
02:10:38.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster2513987901329325466/master-0/data/info.pb
02:10:38.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:10:38.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:10:38.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:10:38.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:10:38.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:10:38.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:10:38.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:10:38.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:10:38.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:10:38.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:10:38.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:10:38.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:10:38.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.63.62:35377
02:10:38.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.63.62
02:10:38.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:10:38.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.63.20:35311
02:10:38.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:10:38.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:10:38.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:10:38.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:10:38.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.30.63.62:35377,127.30.63.61:39749,127.30.63.60:46699
02:10:38.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:10:38.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33017/jwks.json with env {}
02:10:39.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:39.276645 30984 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:10:39.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:39.277195 30984 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:10:39.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:39.277678 30984 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:10:39.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:39.310575 30984 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
02:10:39.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:39.310899 30984 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:10:39.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:39.311110 30984 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
02:10:39.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:39.311314 30984 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
02:10:39.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:39.311513 30984 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
02:10:39.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:39.311663 30984 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33017/jwks.json
02:10:39.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:39.349558 30984 master_runner.cc:386] Master server non-default flags:
02:10:39.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:10:39.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.63.20:35311
02:10:39.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:10:39.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:10:39.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster2513987901329325466/master-0/data
02:10:39.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster2513987901329325466/master-0/wal
02:10:39.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:10:39.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.30.63.62:35377,127.30.63.61:39749,127.30.63.60:46699
02:10:39.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:10:39.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:10:39.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:10:39.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.63.62:35377
02:10:39.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:10:39.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:10:39.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:10:39.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33017/jwks.json
02:10:39.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:10:39.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:10:39.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster2513987901329325466/master-0/data/info.pb
02:10:39.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.63.62
02:10:39.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:10:39.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:10:39.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:10:39.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:10:39.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:10:39.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:10:39.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster2513987901329325466/master-0/logs
02:10:39.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:10:39.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:10:39.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:10:39.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:10:39.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:10:39.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 5688a9afc53de46b2b2f6847bc5217f38e926fe8
02:10:39.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:10:39.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 24 Oct 2025 01:43:20 UTC on 5fd53c4cbb9d
02:10:39.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8689
02:10:39.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:10:39.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:39.350930 30984 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:10:39.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:39.352520 30984 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:10:39.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:39.363845 30990 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:10:41.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:40.768805 30989 debug-util.cc:398] Leaking SignalData structure 0x7b0800037520 after lost signal to thread 30984
02:10:41.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:41.197178 30984 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.833s user 0.684s sys 1.146s
02:10:41.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:41.197551 30984 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.833s user 0.684s sys 1.146s
02:10:41.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:39.364665 30991 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:10:41.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:41.199705 30993 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:10:41.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:41.202256 30992 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1832 milliseconds
02:10:41.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:41.202286 30984 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:10:41.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:41.203728 30984 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:10:41.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:41.206125 30984 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:10:41.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:41.207511 30984 hybrid_clock.cc:648] HybridClock initialized: now 1761271841207473 us; error 40 us; skew 500 ppm
02:10:41.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:41.208375 30984 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:10:41.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:41.218379 30984 webserver.cc:492] Webserver started at http://127.30.63.62:42619/ using document root <none> and password file <none>
02:10:41.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:41.219398 30984 fs_manager.cc:362] Metadata directory not provided
02:10:41.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:41.219621 30984 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:10:41.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:41.220103 30984 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
02:10:41.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:41.224524 30984 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster2513987901329325466/master-0/data/instance:
02:10:41.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "72177680c3ff4418ada40fd5f7ef31da"
02:10:41.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-24 02:10:41 on dist-test-slave-xv5f"
02:10:41.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:41.225697 30984 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster2513987901329325466/master-0/wal/instance:
02:10:41.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "72177680c3ff4418ada40fd5f7ef31da"
02:10:41.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-24 02:10:41 on dist-test-slave-xv5f"
02:10:41.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:41.233234 30984 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.008s sys 0.001s
02:10:41.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:41.239217 31000 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:10:41.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:41.240346 30984 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.000s sys 0.004s
02:10:41.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:41.240754 30984 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster2513987901329325466/master-0/data,/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster2513987901329325466/master-0/wal
02:10:41.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "72177680c3ff4418ada40fd5f7ef31da"
02:10:41.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-24 02:10:41 on dist-test-slave-xv5f"
02:10:41.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:41.241117 30984 fs_report.cc:389] FS layout report
02:10:41.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:10:41.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster2513987901329325466/master-0/wal
02:10:41.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster2513987901329325466/master-0/wal
02:10:41.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster2513987901329325466/master-0/data/data
02:10:41.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:10:41.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:10:41.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:10:41.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:10:41.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:10:41.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:10:41.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:10:41.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:10:41.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:10:41.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:10:41.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:10:41.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:41.325649 30984 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:10:41.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:41.327488 30984 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:10:41.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:41.328162 30984 kserver.cc:163] Server-wide thread pool size limit: 3276
02:10:41.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:41.414945 30984 rpc_server.cc:307] RPC server started. Bound to: 127.30.63.62:35377
02:10:41.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:41.415005 31051 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.30.63.62:35377 every 8 connection(s)
02:10:41.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:41.417910 30984 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster2513987901329325466/master-0/data/info.pb
02:10:41.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:41.424146 31052 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
02:10:41.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:41.428153 30972 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskOg0PFu/build/tsan/bin/kudu as pid 30984
02:10:41.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:41.428552 30972 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster2513987901329325466/master-0/wal/instance
02:10:41.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:41.431993 30972 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskOg0PFu/build/tsan/bin/kudu
02:10:41.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskOg0PFu/build/tsan/bin/kudu
02:10:41.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster2513987901329325466/master-1/wal
02:10:41.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster2513987901329325466/master-1/data
02:10:41.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:10:41.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:10:41.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:10:41.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:10:41.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:10:41.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:10:41.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster2513987901329325466/master-1/logs
02:10:41.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster2513987901329325466/master-1/data/info.pb
02:10:41.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:10:41.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:10:41.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:10:41.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:10:41.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:10:41.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:10:41.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:10:41.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:10:41.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:10:41.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:10:41.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:10:41.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:10:41.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.63.61:39749
02:10:41.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.63.61
02:10:41.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:10:41.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.63.20:35311
02:10:41.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:10:41.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:10:41.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:10:41.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:10:41.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.30.63.62:35377,127.30.63.61:39749,127.30.63.60:46699
02:10:41.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:10:41.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33017/jwks.json with env {}
02:10:41.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:41.436123 31052 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.30.63.62" port: 35377 } has no permanent_uuid. Determining permanent_uuid...
02:10:41.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:41.463639 31052 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.30.63.61" port: 39749 } has no permanent_uuid. Determining permanent_uuid...
02:10:41.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:41.466182 31001 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.30.63.61:39749: connect: Connection refused (error 111)
02:10:41.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:41.470587 31052 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.30.63.61:39749: Network error: Client connection negotiation failed: client connection to 127.30.63.61:39749: connect: Connection refused (error 111)
02:10:41.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:41.520272 31052 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.63.61" port: 39749 } attempt: 1
02:10:41.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:41.524441 31052 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.30.63.61:39749: Network error: Client connection negotiation failed: client connection to 127.30.63.61:39749: connect: Connection refused (error 111)
02:10:41.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:41.593003 31052 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.63.61" port: 39749 } attempt: 2
02:10:41.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:41.597093 31052 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.30.63.61:39749: Network error: Client connection negotiation failed: client connection to 127.30.63.61:39749: connect: Connection refused (error 111)
02:10:41.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:41.688608 31052 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.63.61" port: 39749 } attempt: 3
02:10:41.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:41.692672 31052 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.30.63.61:39749: Network error: Client connection negotiation failed: client connection to 127.30.63.61:39749: connect: Connection refused (error 111)
02:10:41.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:41.743834 31054 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:10:41.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:41.744491 31054 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:10:41.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:41.745066 31054 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:10:41.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:41.778077 31054 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
02:10:41.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:41.778467 31054 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:10:41.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:41.778789 31054 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
02:10:41.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:41.779058 31054 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
02:10:41.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:41.779322 31054 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
02:10:41.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:41.779543 31054 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33017/jwks.json
02:10:41.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:41.817139 31054 master_runner.cc:386] Master server non-default flags:
02:10:41.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:10:41.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.63.20:35311
02:10:41.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:10:41.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:10:41.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster2513987901329325466/master-1/data
02:10:41.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster2513987901329325466/master-1/wal
02:10:41.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:10:41.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.30.63.62:35377,127.30.63.61:39749,127.30.63.60:46699
02:10:41.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:10:41.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:10:41.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:10:41.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.63.61:39749
02:10:41.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:10:41.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:10:41.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:10:41.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33017/jwks.json
02:10:41.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:10:41.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:10:41.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster2513987901329325466/master-1/data/info.pb
02:10:41.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.63.61
02:10:41.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:10:41.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:10:41.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:10:41.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:10:41.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:10:41.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:10:41.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster2513987901329325466/master-1/logs
02:10:41.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:10:41.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:10:41.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:10:41.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:10:41.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:10:41.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 5688a9afc53de46b2b2f6847bc5217f38e926fe8
02:10:41.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:10:41.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 24 Oct 2025 01:43:20 UTC on 5fd53c4cbb9d
02:10:41.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8689
02:10:41.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:10:41.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:41.818849 31054 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:10:41.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:41.820567 31054 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:10:41.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:41.832034 31063 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:10:41.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:41.836117 31052 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.63.61" port: 39749 } attempt: 4
02:10:41.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:41.841641 31052 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.30.63.61:39749: Network error: Client connection negotiation failed: client connection to 127.30.63.61:39749: connect: Connection refused (error 111)
02:10:42.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:42.141142 31052 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.63.61" port: 39749 } attempt: 5
02:10:42.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:42.148131 31052 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.30.63.61:39749: Network error: Client connection negotiation failed: client connection to 127.30.63.61:39749: connect: Connection refused (error 111)
02:10:42.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:42.695791 31052 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.63.61" port: 39749 } attempt: 6
02:10:42.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:42.710917 31052 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.30.63.61:39749: Network error: Client connection negotiation failed: client connection to 127.30.63.61:39749: connect: Connection refused (error 111)
02:10:43.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:41.832384 31064 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:10:43.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:41.836253 31066 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:10:43.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:43.089377 31065 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1252 milliseconds
02:10:43.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:43.089524 31054 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:10:43.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:43.090925 31054 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:10:43.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:43.093504 31054 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:10:43.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:43.094868 31054 hybrid_clock.cc:648] HybridClock initialized: now 1761271843094821 us; error 55 us; skew 500 ppm
02:10:43.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:43.095705 31054 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:10:43.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:43.102293 31054 webserver.cc:492] Webserver started at http://127.30.63.61:44831/ using document root <none> and password file <none>
02:10:43.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:43.103323 31054 fs_manager.cc:362] Metadata directory not provided
02:10:43.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:43.103552 31054 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:10:43.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:43.104075 31054 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
02:10:43.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:43.108414 31054 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster2513987901329325466/master-1/data/instance:
02:10:43.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "47c8574df61247388b26dff25f225818"
02:10:43.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-24 02:10:43 on dist-test-slave-xv5f"
02:10:43.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:43.109558 31054 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster2513987901329325466/master-1/wal/instance:
02:10:43.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "47c8574df61247388b26dff25f225818"
02:10:43.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-24 02:10:43 on dist-test-slave-xv5f"
02:10:43.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:43.116739 31054 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.009s sys 0.000s
02:10:43.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:43.122510 31075 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:10:43.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:43.123569 31054 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.004s sys 0.000s
02:10:43.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:43.123916 31054 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster2513987901329325466/master-1/data,/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster2513987901329325466/master-1/wal
02:10:43.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "47c8574df61247388b26dff25f225818"
02:10:43.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-24 02:10:43 on dist-test-slave-xv5f"
02:10:43.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:43.124322 31054 fs_report.cc:389] FS layout report
02:10:43.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:10:43.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster2513987901329325466/master-1/wal
02:10:43.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster2513987901329325466/master-1/wal
02:10:43.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster2513987901329325466/master-1/data/data
02:10:43.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:10:43.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:10:43.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:10:43.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:10:43.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:10:43.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:10:43.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:10:43.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:10:43.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:10:43.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:10:43.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:10:43.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:43.176268 31054 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:10:43.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:43.177881 31054 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:10:43.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:43.178367 31054 kserver.cc:163] Server-wide thread pool size limit: 3276
02:10:43.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:43.252894 31054 rpc_server.cc:307] RPC server started. Bound to: 127.30.63.61:39749
02:10:43.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:43.252943 31126 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.30.63.61:39749 every 8 connection(s)
02:10:43.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:43.255919 31054 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster2513987901329325466/master-1/data/info.pb
02:10:43.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:43.260099 30972 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskOg0PFu/build/tsan/bin/kudu as pid 31054
02:10:43.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:43.260653 30972 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster2513987901329325466/master-1/wal/instance
02:10:43.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:43.263403 31127 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
02:10:43.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:43.263702 30972 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskOg0PFu/build/tsan/bin/kudu
02:10:43.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskOg0PFu/build/tsan/bin/kudu
02:10:43.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster2513987901329325466/master-2/wal
02:10:43.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster2513987901329325466/master-2/data
02:10:43.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:10:43.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:10:43.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:10:43.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:10:43.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:10:43.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:10:43.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster2513987901329325466/master-2/logs
02:10:43.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster2513987901329325466/master-2/data/info.pb
02:10:43.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:10:43.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:10:43.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:10:43.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:10:43.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:10:43.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:10:43.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:10:43.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:10:43.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:10:43.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:10:43.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:10:43.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:10:43.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.63.60:46699
02:10:43.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.63.60
02:10:43.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:10:43.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.63.20:35311
02:10:43.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:10:43.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:10:43.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:10:43.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:10:43.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.30.63.62:35377,127.30.63.61:39749,127.30.63.60:46699
02:10:43.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:10:43.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33017/jwks.json with env {}
02:10:43.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:43.277586 31127 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.30.63.62" port: 35377 } has no permanent_uuid. Determining permanent_uuid...
02:10:43.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:43.300865 31127 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.30.63.61" port: 39749 } has no permanent_uuid. Determining permanent_uuid...
02:10:43.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:43.316190 31127 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.30.63.60" port: 46699 } has no permanent_uuid. Determining permanent_uuid...
02:10:43.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:43.319049 31078 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.30.63.60:46699: connect: Connection refused (error 111)
02:10:43.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:43.323345 31127 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.30.63.60:46699: Network error: Client connection negotiation failed: client connection to 127.30.63.60:46699: connect: Connection refused (error 111)
02:10:43.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:43.372849 31127 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.63.60" port: 46699 } attempt: 1
02:10:43.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:43.376953 31127 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.30.63.60:46699: Network error: Client connection negotiation failed: client connection to 127.30.63.60:46699: connect: Connection refused (error 111)
02:10:43.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:43.445370 31127 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.63.60" port: 46699 } attempt: 2
02:10:43.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:43.449759 31127 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.30.63.60:46699: Network error: Client connection negotiation failed: client connection to 127.30.63.60:46699: connect: Connection refused (error 111)
02:10:43.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:43.541256 31127 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.63.60" port: 46699 } attempt: 3
02:10:43.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:43.545357 31127 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.30.63.60:46699: Network error: Client connection negotiation failed: client connection to 127.30.63.60:46699: connect: Connection refused (error 111)
02:10:43.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:43.583448 31129 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:10:43.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:43.584105 31129 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:10:43.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:43.584649 31129 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:10:43.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:43.617925 31129 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
02:10:43.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:43.618299 31129 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:10:43.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:43.618647 31129 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
02:10:43.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:43.618943 31129 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
02:10:43.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:43.619231 31129 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
02:10:43.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:43.619447 31129 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33017/jwks.json
02:10:43.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:43.656786 31129 master_runner.cc:386] Master server non-default flags:
02:10:43.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:10:43.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.63.20:35311
02:10:43.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:10:43.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:10:43.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster2513987901329325466/master-2/data
02:10:43.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster2513987901329325466/master-2/wal
02:10:43.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:10:43.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.30.63.62:35377,127.30.63.61:39749,127.30.63.60:46699
02:10:43.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:10:43.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:10:43.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:10:43.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.63.60:46699
02:10:43.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:10:43.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:10:43.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:10:43.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33017/jwks.json
02:10:43.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:10:43.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:10:43.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster2513987901329325466/master-2/data/info.pb
02:10:43.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.63.60
02:10:43.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:10:43.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:10:43.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:10:43.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:10:43.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:10:43.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:10:43.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster2513987901329325466/master-2/logs
02:10:43.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:10:43.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:10:43.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:10:43.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:10:43.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:10:43.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 5688a9afc53de46b2b2f6847bc5217f38e926fe8
02:10:43.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:10:43.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 24 Oct 2025 01:43:20 UTC on 5fd53c4cbb9d
02:10:43.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8689
02:10:43.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:10:43.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:43.658433 31129 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:10:43.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:43.660193 31129 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:10:43.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:43.671934 31139 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:10:43.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:43.688766 31127 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.63.60" port: 46699 } attempt: 4
02:10:43.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:43.694114 31127 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.30.63.60:46699: Network error: Client connection negotiation failed: client connection to 127.30.63.60:46699: connect: Connection refused (error 111)
02:10:43.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:43.777829 31052 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.63.61" port: 39749 } attempt: 7
02:10:43.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:43.812224 31052 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.30.63.60" port: 46699 } has no permanent_uuid. Determining permanent_uuid...
02:10:43.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:43.833520 31052 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.30.63.60:46699: Network error: Client connection negotiation failed: client connection to 127.30.63.60:46699: connect: Connection refused (error 111)
02:10:43.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:43.892097 31052 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.63.60" port: 46699 } attempt: 1
02:10:43.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:43.899655 31052 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.30.63.60:46699: Network error: Client connection negotiation failed: client connection to 127.30.63.60:46699: connect: Connection refused (error 111)
02:10:43.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:43.981230 31052 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.63.60" port: 46699 } attempt: 2
02:10:43.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:43.991588 31052 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.30.63.60:46699: Network error: Client connection negotiation failed: client connection to 127.30.63.60:46699: connect: Connection refused (error 111)
02:10:43.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:43.993706 31127 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.63.60" port: 46699 } attempt: 5
02:10:44.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:44.001430 31127 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.30.63.60:46699: Network error: Client connection negotiation failed: client connection to 127.30.63.60:46699: connect: Connection refused (error 111)
02:10:44.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:44.055099 31048 debug-util.cc:398] Leaking SignalData structure 0x7b080007be80 after lost signal to thread 30985
02:10:44.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:44.077143 31052 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.63.60" port: 46699 } attempt: 3
02:10:44.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:44.085067 31052 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.30.63.60:46699: Network error: Client connection negotiation failed: client connection to 127.30.63.60:46699: connect: Connection refused (error 111)
02:10:44.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:44.225622 31052 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.63.60" port: 46699 } attempt: 4
02:10:44.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:44.234061 31052 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.30.63.60:46699: Network error: Client connection negotiation failed: client connection to 127.30.63.60:46699: connect: Connection refused (error 111)
02:10:44.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:44.517647 31052 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.63.60" port: 46699 } attempt: 5
02:10:44.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:44.524464 31052 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.30.63.60:46699: Network error: Client connection negotiation failed: client connection to 127.30.63.60:46699: connect: Connection refused (error 111)
02:10:44.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:44.549145 31127 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.63.60" port: 46699 } attempt: 6
02:10:44.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:44.565726 31127 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.30.63.60:46699: Network error: Client connection negotiation failed: client connection to 127.30.63.60:46699: connect: Connection refused (error 111)
02:10:45.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:45.076900 31052 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.63.60" port: 46699 } attempt: 6
02:10:45.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:45.109771 31052 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.30.63.60:46699: Network error: Client connection negotiation failed: client connection to 127.30.63.60:46699: connect: Connection refused (error 111)
02:10:45.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:45.075455 31138 debug-util.cc:398] Leaking SignalData structure 0x7b0800037520 after lost signal to thread 31129
02:10:45.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:45.522071 31129 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.850s user 0.000s sys 0.002s
02:10:45.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:45.522646 31129 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.851s user 0.001s sys 0.002s
02:10:45.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:45.523452 31141 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1848 milliseconds
02:10:45.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:43.672465 31140 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:10:45.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:45.524593 31142 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:10:45.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:45.524549 31129 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:10:45.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:45.529433 31129 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:10:45.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:45.532446 31129 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:10:45.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:45.533921 31129 hybrid_clock.cc:648] HybridClock initialized: now 1761271845533864 us; error 73 us; skew 500 ppm
02:10:45.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:45.534806 31129 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:10:45.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:45.542557 31129 webserver.cc:492] Webserver started at http://127.30.63.60:34169/ using document root <none> and password file <none>
02:10:45.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:45.543495 31129 fs_manager.cc:362] Metadata directory not provided
02:10:45.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:45.543689 31129 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:10:45.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:45.544101 31129 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
02:10:45.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:45.548441 31129 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster2513987901329325466/master-2/data/instance:
02:10:45.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4672000d27184e65a1eb4f4a11332b56"
02:10:45.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-24 02:10:45 on dist-test-slave-xv5f"
02:10:45.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:45.549517 31129 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster2513987901329325466/master-2/wal/instance:
02:10:45.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4672000d27184e65a1eb4f4a11332b56"
02:10:45.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-24 02:10:45 on dist-test-slave-xv5f"
02:10:45.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:45.557580 31129 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.005s sys 0.004s
02:10:45.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:45.564252 31155 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:10:45.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:45.565486 31129 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.003s
02:10:45.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:45.565836 31129 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster2513987901329325466/master-2/data,/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster2513987901329325466/master-2/wal
02:10:45.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4672000d27184e65a1eb4f4a11332b56"
02:10:45.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-24 02:10:45 on dist-test-slave-xv5f"
02:10:45.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:45.566156 31129 fs_report.cc:389] FS layout report
02:10:45.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:10:45.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster2513987901329325466/master-2/wal
02:10:45.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster2513987901329325466/master-2/wal
02:10:45.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster2513987901329325466/master-2/data/data
02:10:45.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:10:45.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:10:45.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:10:45.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:10:45.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:10:45.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:10:45.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:10:45.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:10:45.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:10:45.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:10:45.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:10:45.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:45.626330 31127 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.63.60" port: 46699 } attempt: 7
02:10:45.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:45.631373 31129 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:10:45.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:45.633716 31129 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:10:45.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:45.634454 31129 kserver.cc:163] Server-wide thread pool size limit: 3276
02:10:45.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:45.635156 31127 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.30.63.60:46699: Network error: Client connection negotiation failed: client connection to 127.30.63.60:46699: connect: Connection refused (error 111)
02:10:45.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:45.727844 31129 rpc_server.cc:307] RPC server started. Bound to: 127.30.63.60:46699
02:10:45.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:45.727907 31208 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.30.63.60:46699 every 8 connection(s)
02:10:45.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:45.730737 31129 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster2513987901329325466/master-2/data/info.pb
02:10:45.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:45.732316 30972 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskOg0PFu/build/tsan/bin/kudu as pid 31129
02:10:45.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:45.732818 30972 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster2513987901329325466/master-2/wal/instance
02:10:45.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:45.740211 31209 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
02:10:45.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:45.764617 31209 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.30.63.62" port: 35377 } has no permanent_uuid. Determining permanent_uuid...
02:10:45.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:45.818504 31209 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.30.63.61" port: 39749 } has no permanent_uuid. Determining permanent_uuid...
02:10:45.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:45.832227 31209 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.30.63.60" port: 46699 } has no permanent_uuid. Determining permanent_uuid...
02:10:45.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:45.853106 31209 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4672000d27184e65a1eb4f4a11332b56: Bootstrap starting.
02:10:45.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:45.861068 31209 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 4672000d27184e65a1eb4f4a11332b56: Neither blocks nor log segments found. Creating new log.
02:10:45.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:45.862995 31209 log.cc:826] T 00000000000000000000000000000000 P 4672000d27184e65a1eb4f4a11332b56: Log is configured to *not* fsync() on all Append() calls
02:10:45.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:45.868750 31209 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4672000d27184e65a1eb4f4a11332b56: No bootstrap required, opened a new log
02:10:45.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:45.890029 31209 raft_consensus.cc:359] T 00000000000000000000000000000000 P 4672000d27184e65a1eb4f4a11332b56 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "72177680c3ff4418ada40fd5f7ef31da" member_type: VOTER last_known_addr { host: "127.30.63.62" port: 35377 } } peers { permanent_uuid: "47c8574df61247388b26dff25f225818" member_type: VOTER last_known_addr { host: "127.30.63.61" port: 39749 } } peers { permanent_uuid: "4672000d27184e65a1eb4f4a11332b56" member_type: VOTER last_known_addr { host: "127.30.63.60" port: 46699 } }
02:10:45.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:45.891505 31209 raft_consensus.cc:385] T 00000000000000000000000000000000 P 4672000d27184e65a1eb4f4a11332b56 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:10:45.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:45.891996 31209 raft_consensus.cc:740] T 00000000000000000000000000000000 P 4672000d27184e65a1eb4f4a11332b56 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4672000d27184e65a1eb4f4a11332b56, State: Initialized, Role: FOLLOWER
02:10:45.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:45.892971 31209 consensus_queue.cc:260] T 00000000000000000000000000000000 P 4672000d27184e65a1eb4f4a11332b56 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "72177680c3ff4418ada40fd5f7ef31da" member_type: VOTER last_known_addr { host: "127.30.63.62" port: 35377 } } peers { permanent_uuid: "47c8574df61247388b26dff25f225818" member_type: VOTER last_known_addr { host: "127.30.63.61" port: 39749 } } peers { permanent_uuid: "4672000d27184e65a1eb4f4a11332b56" member_type: VOTER last_known_addr { host: "127.30.63.60" port: 46699 } }
02:10:45.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:45.896219 31220 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4672000d27184e65a1eb4f4a11332b56 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "72177680c3ff4418ada40fd5f7ef31da" member_type: VOTER last_known_addr { host: "127.30.63.62" port: 35377 } } peers { permanent_uuid: "47c8574df61247388b26dff25f225818" member_type: VOTER last_known_addr { host: "127.30.63.61" port: 39749 } } peers { permanent_uuid: "4672000d27184e65a1eb4f4a11332b56" member_type: VOTER last_known_addr { host: "127.30.63.60" port: 46699 } } }
02:10:45.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:45.897161 31220 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4672000d27184e65a1eb4f4a11332b56 [sys.catalog]: This master's current role is: FOLLOWER
02:10:45.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:45.898247 31209 sys_catalog.cc:565] T 00000000000000000000000000000000 P 4672000d27184e65a1eb4f4a11332b56 [sys.catalog]: configured and running, proceeding with master startup.
02:10:45.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:45.932809 31231 catalog_manager.cc:1568] T 00000000000000000000000000000000 P 4672000d27184e65a1eb4f4a11332b56: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:10:45.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:45.933475 31231 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:10:45.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:45.926637 31129 master_runner.cc:186] Error getting master registration for 127.30.63.62:35377: OK, instance_id { permanent_uuid: "72177680c3ff4418ada40fd5f7ef31da" instance_seqno: 1761271841395255 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
02:10:45.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:45.963093 31129 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
02:10:46.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:46.141153 31163 tablet.cc:2378] T 00000000000000000000000000000000 P 4672000d27184e65a1eb4f4a11332b56: Can't schedule compaction. Clean time has not been advanced past its initial value.
02:10:46.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:46.143268 31052 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.63.60" port: 46699 } attempt: 7
02:10:46.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:46.168318 31052 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 72177680c3ff4418ada40fd5f7ef31da: Bootstrap starting.
02:10:46.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:46.176548 31052 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 72177680c3ff4418ada40fd5f7ef31da: Neither blocks nor log segments found. Creating new log.
02:10:46.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:46.179090 31052 log.cc:826] T 00000000000000000000000000000000 P 72177680c3ff4418ada40fd5f7ef31da: Log is configured to *not* fsync() on all Append() calls
02:10:46.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:46.184773 31052 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 72177680c3ff4418ada40fd5f7ef31da: No bootstrap required, opened a new log
02:10:46.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:46.214074 31220 raft_consensus.cc:493] T 00000000000000000000000000000000 P 4672000d27184e65a1eb4f4a11332b56 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:10:46.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:46.214744 31220 raft_consensus.cc:515] T 00000000000000000000000000000000 P 4672000d27184e65a1eb4f4a11332b56 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "72177680c3ff4418ada40fd5f7ef31da" member_type: VOTER last_known_addr { host: "127.30.63.62" port: 35377 } } peers { permanent_uuid: "47c8574df61247388b26dff25f225818" member_type: VOTER last_known_addr { host: "127.30.63.61" port: 39749 } } peers { permanent_uuid: "4672000d27184e65a1eb4f4a11332b56" member_type: VOTER last_known_addr { host: "127.30.63.60" port: 46699 } }
02:10:46.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:46.217453 31220 leader_election.cc:290] T 00000000000000000000000000000000 P 4672000d27184e65a1eb4f4a11332b56 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 72177680c3ff4418ada40fd5f7ef31da (127.30.63.62:35377), 47c8574df61247388b26dff25f225818 (127.30.63.61:39749)
02:10:46.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:46.223203 31052 raft_consensus.cc:359] T 00000000000000000000000000000000 P 72177680c3ff4418ada40fd5f7ef31da [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "72177680c3ff4418ada40fd5f7ef31da" member_type: VOTER last_known_addr { host: "127.30.63.62" port: 35377 } } peers { permanent_uuid: "47c8574df61247388b26dff25f225818" member_type: VOTER last_known_addr { host: "127.30.63.61" port: 39749 } } peers { permanent_uuid: "4672000d27184e65a1eb4f4a11332b56" member_type: VOTER last_known_addr { host: "127.30.63.60" port: 46699 } }
02:10:46.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:46.225128 31052 raft_consensus.cc:385] T 00000000000000000000000000000000 P 72177680c3ff4418ada40fd5f7ef31da [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:10:46.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:46.225867 31052 raft_consensus.cc:740] T 00000000000000000000000000000000 P 72177680c3ff4418ada40fd5f7ef31da [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 72177680c3ff4418ada40fd5f7ef31da, State: Initialized, Role: FOLLOWER
02:10:46.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:46.227847 31052 consensus_queue.cc:260] T 00000000000000000000000000000000 P 72177680c3ff4418ada40fd5f7ef31da [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "72177680c3ff4418ada40fd5f7ef31da" member_type: VOTER last_known_addr { host: "127.30.63.62" port: 35377 } } peers { permanent_uuid: "47c8574df61247388b26dff25f225818" member_type: VOTER last_known_addr { host: "127.30.63.61" port: 39749 } } peers { permanent_uuid: "4672000d27184e65a1eb4f4a11332b56" member_type: VOTER last_known_addr { host: "127.30.63.60" port: 46699 } }
02:10:46.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:46.233543 31235 sys_catalog.cc:455] T 00000000000000000000000000000000 P 72177680c3ff4418ada40fd5f7ef31da [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "72177680c3ff4418ada40fd5f7ef31da" member_type: VOTER last_known_addr { host: "127.30.63.62" port: 35377 } } peers { permanent_uuid: "47c8574df61247388b26dff25f225818" member_type: VOTER last_known_addr { host: "127.30.63.61" port: 39749 } } peers { permanent_uuid: "4672000d27184e65a1eb4f4a11332b56" member_type: VOTER last_known_addr { host: "127.30.63.60" port: 46699 } } }
02:10:46.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:46.235562 31235 sys_catalog.cc:458] T 00000000000000000000000000000000 P 72177680c3ff4418ada40fd5f7ef31da [sys.catalog]: This master's current role is: FOLLOWER
02:10:46.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:46.237601 31027 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "4672000d27184e65a1eb4f4a11332b56" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "72177680c3ff4418ada40fd5f7ef31da" is_pre_election: true
02:10:46.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:46.238623 31027 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 72177680c3ff4418ada40fd5f7ef31da [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 4672000d27184e65a1eb4f4a11332b56 in term 0.
02:10:46.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:46.240548 31159 leader_election.cc:304] T 00000000000000000000000000000000 P 4672000d27184e65a1eb4f4a11332b56 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4672000d27184e65a1eb4f4a11332b56, 72177680c3ff4418ada40fd5f7ef31da; no voters:
02:10:46.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:46.241591 31220 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 4672000d27184e65a1eb4f4a11332b56 [term 0 FOLLOWER]: Leader pre-election won for term 1
02:10:46.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:46.241958 31220 raft_consensus.cc:493] T 00000000000000000000000000000000 P 4672000d27184e65a1eb4f4a11332b56 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
02:10:46.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:46.242296 31220 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 4672000d27184e65a1eb4f4a11332b56 [term 0 FOLLOWER]: Advancing to term 1
02:10:46.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:46.259249 31220 raft_consensus.cc:515] T 00000000000000000000000000000000 P 4672000d27184e65a1eb4f4a11332b56 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "72177680c3ff4418ada40fd5f7ef31da" member_type: VOTER last_known_addr { host: "127.30.63.62" port: 35377 } } peers { permanent_uuid: "47c8574df61247388b26dff25f225818" member_type: VOTER last_known_addr { host: "127.30.63.61" port: 39749 } } peers { permanent_uuid: "4672000d27184e65a1eb4f4a11332b56" member_type: VOTER last_known_addr { host: "127.30.63.60" port: 46699 } }
02:10:46.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:46.262562 31052 sys_catalog.cc:565] T 00000000000000000000000000000000 P 72177680c3ff4418ada40fd5f7ef31da [sys.catalog]: configured and running, proceeding with master startup.
02:10:46.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:46.265643 31027 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "4672000d27184e65a1eb4f4a11332b56" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "72177680c3ff4418ada40fd5f7ef31da"
02:10:46.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:46.266299 31027 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 72177680c3ff4418ada40fd5f7ef31da [term 0 FOLLOWER]: Advancing to term 1
02:10:46.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:46.274019 31027 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 72177680c3ff4418ada40fd5f7ef31da [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 4672000d27184e65a1eb4f4a11332b56 in term 1.
02:10:46.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:46.278935 31220 leader_election.cc:290] T 00000000000000000000000000000000 P 4672000d27184e65a1eb4f4a11332b56 [CANDIDATE]: Term 1 election: Requested vote from peers 72177680c3ff4418ada40fd5f7ef31da (127.30.63.62:35377), 47c8574df61247388b26dff25f225818 (127.30.63.61:39749)
02:10:46.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:46.280296 31159 leader_election.cc:304] T 00000000000000000000000000000000 P 4672000d27184e65a1eb4f4a11332b56 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4672000d27184e65a1eb4f4a11332b56, 72177680c3ff4418ada40fd5f7ef31da; no voters:
02:10:46.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:46.287829 31220 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 4672000d27184e65a1eb4f4a11332b56 [term 1 FOLLOWER]: Leader election won for term 1
02:10:46.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:46.295092 31220 raft_consensus.cc:697] T 00000000000000000000000000000000 P 4672000d27184e65a1eb4f4a11332b56 [term 1 LEADER]: Becoming Leader. State: Replica: 4672000d27184e65a1eb4f4a11332b56, State: Running, Role: LEADER
02:10:46.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:46.297888 31220 consensus_queue.cc:237] T 00000000000000000000000000000000 P 4672000d27184e65a1eb4f4a11332b56 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "72177680c3ff4418ada40fd5f7ef31da" member_type: VOTER last_known_addr { host: "127.30.63.62" port: 35377 } } peers { permanent_uuid: "47c8574df61247388b26dff25f225818" member_type: VOTER last_known_addr { host: "127.30.63.61" port: 39749 } } peers { permanent_uuid: "4672000d27184e65a1eb4f4a11332b56" member_type: VOTER last_known_addr { host: "127.30.63.60" port: 46699 } }
02:10:46.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:46.319382 31236 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4672000d27184e65a1eb4f4a11332b56 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 4672000d27184e65a1eb4f4a11332b56. Latest consensus state: current_term: 1 leader_uuid: "4672000d27184e65a1eb4f4a11332b56" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "72177680c3ff4418ada40fd5f7ef31da" member_type: VOTER last_known_addr { host: "127.30.63.62" port: 35377 } } peers { permanent_uuid: "47c8574df61247388b26dff25f225818" member_type: VOTER last_known_addr { host: "127.30.63.61" port: 39749 } } peers { permanent_uuid: "4672000d27184e65a1eb4f4a11332b56" member_type: VOTER last_known_addr { host: "127.30.63.60" port: 46699 } } }
02:10:46.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:46.320433 31236 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4672000d27184e65a1eb4f4a11332b56 [sys.catalog]: This master's current role is: LEADER
02:10:46.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:46.325966 31241 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
02:10:46.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:46.341833 31006 tablet.cc:2378] T 00000000000000000000000000000000 P 72177680c3ff4418ada40fd5f7ef31da: Can't schedule compaction. Clean time has not been advanced past its initial value.
02:10:46.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:46.358227 31241 catalog_manager.cc:1494] Initializing Kudu cluster ID...
02:10:46.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:46.409191 31027 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 72177680c3ff4418ada40fd5f7ef31da [term 1 FOLLOWER]: Refusing update from remote peer 4672000d27184e65a1eb4f4a11332b56: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
02:10:46.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:46.411502 31236 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 4672000d27184e65a1eb4f4a11332b56 [LEADER]: Connected to new peer: Peer: permanent_uuid: "72177680c3ff4418ada40fd5f7ef31da" member_type: VOTER last_known_addr { host: "127.30.63.62" port: 35377 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
02:10:46.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:46.447928 31235 sys_catalog.cc:455] T 00000000000000000000000000000000 P 72177680c3ff4418ada40fd5f7ef31da [sys.catalog]: SysCatalogTable state changed. Reason: New leader 4672000d27184e65a1eb4f4a11332b56. Latest consensus state: current_term: 1 leader_uuid: "4672000d27184e65a1eb4f4a11332b56" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "72177680c3ff4418ada40fd5f7ef31da" member_type: VOTER last_known_addr { host: "127.30.63.62" port: 35377 } } peers { permanent_uuid: "47c8574df61247388b26dff25f225818" member_type: VOTER last_known_addr { host: "127.30.63.61" port: 39749 } } peers { permanent_uuid: "4672000d27184e65a1eb4f4a11332b56" member_type: VOTER last_known_addr { host: "127.30.63.60" port: 46699 } } }
02:10:46.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:46.448766 31235 sys_catalog.cc:458] T 00000000000000000000000000000000 P 72177680c3ff4418ada40fd5f7ef31da [sys.catalog]: This master's current role is: FOLLOWER
02:10:46.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:46.461866 31246 mvcc.cc:204] Tried to move back new op lower bound from 7214169482861649920 to 7214169482478379008. Current Snapshot: MvccSnapshot[applied={T|T < 7214169482861649920}]
02:10:46.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:46.485255 31235 sys_catalog.cc:455] T 00000000000000000000000000000000 P 72177680c3ff4418ada40fd5f7ef31da [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "4672000d27184e65a1eb4f4a11332b56" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "72177680c3ff4418ada40fd5f7ef31da" member_type: VOTER last_known_addr { host: "127.30.63.62" port: 35377 } } peers { permanent_uuid: "47c8574df61247388b26dff25f225818" member_type: VOTER last_known_addr { host: "127.30.63.61" port: 39749 } } peers { permanent_uuid: "4672000d27184e65a1eb4f4a11332b56" member_type: VOTER last_known_addr { host: "127.30.63.60" port: 46699 } } }
02:10:46.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:46.485971 31235 sys_catalog.cc:458] T 00000000000000000000000000000000 P 72177680c3ff4418ada40fd5f7ef31da [sys.catalog]: This master's current role is: FOLLOWER
02:10:46.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:46.523075 31220 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4672000d27184e65a1eb4f4a11332b56 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "4672000d27184e65a1eb4f4a11332b56" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "72177680c3ff4418ada40fd5f7ef31da" member_type: VOTER last_known_addr { host: "127.30.63.62" port: 35377 } } peers { permanent_uuid: "47c8574df61247388b26dff25f225818" member_type: VOTER last_known_addr { host: "127.30.63.61" port: 39749 } } peers { permanent_uuid: "4672000d27184e65a1eb4f4a11332b56" member_type: VOTER last_known_addr { host: "127.30.63.60" port: 46699 } } }
02:10:46.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:46.524094 31220 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4672000d27184e65a1eb4f4a11332b56 [sys.catalog]: This master's current role is: LEADER
02:10:46.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:46.481243 30984 master_runner.cc:186] Error getting master registration for 127.30.63.61:39749: OK, instance_id { permanent_uuid: "47c8574df61247388b26dff25f225818" instance_seqno: 1761271843232457 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
02:10:46.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:46.542673 30984 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
02:10:46.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:46.547384 31241 catalog_manager.cc:1357] Generated new cluster ID: 2594582366af42bfa6861b383a2b4ad5
02:10:46.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:46.547739 31241 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
02:10:46.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:46.552418 31252 catalog_manager.cc:1568] T 00000000000000000000000000000000 P 72177680c3ff4418ada40fd5f7ef31da: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:10:46.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:46.560457 31252 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:10:46.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:46.589690 31241 catalog_manager.cc:1380] Generated new certificate authority record
02:10:46.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:46.600564 31241 catalog_manager.cc:1514] Loading token signing keys...
02:10:46.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:46.624716 31241 catalog_manager.cc:6022] T 00000000000000000000000000000000 P 4672000d27184e65a1eb4f4a11332b56: Generated new TSK 0
02:10:46.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:46.626885 31241 catalog_manager.cc:1524] Initializing in-progress tserver states...
02:10:46.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:46.682682 30972 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskOg0PFu/build/tsan/bin/kudu
02:10:46.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskOg0PFu/build/tsan/bin/kudu
02:10:46.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster2513987901329325466/ts-0/wal
02:10:46.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster2513987901329325466/ts-0/data
02:10:46.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:10:46.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:10:46.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:10:46.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:10:46.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:10:46.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:10:46.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster2513987901329325466/ts-0/logs
02:10:46.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster2513987901329325466/ts-0/data/info.pb
02:10:46.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:10:46.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:10:46.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:10:46.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:10:46.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:10:46.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:10:46.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:10:46.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:10:46.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
02:10:46.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:10:46.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.63.1:0
02:10:46.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.30.63.1
02:10:46.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.63.1
02:10:46.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:10:46.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.30.63.62:35377,127.30.63.61:39749,127.30.63.60:46699
02:10:46.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.63.20:35311
02:10:46.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:10:46.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:10:46.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
02:10:47.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:47.210129 31256 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:10:47.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:47.211067 31256 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:10:47.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:47.211975 31256 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:10:47.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:47.306495 31256 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:10:47.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:47.307837 31256 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.30.63.1
02:10:47.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:47.373267 31256 tablet_server_runner.cc:78] Tablet server non-default flags:
02:10:47.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:10:47.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.63.20:35311
02:10:47.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:10:47.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:10:47.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster2513987901329325466/ts-0/data
02:10:47.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster2513987901329325466/ts-0/wal
02:10:47.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:10:47.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:10:47.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.63.1:0
02:10:47.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:10:47.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:10:47.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:10:47.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster2513987901329325466/ts-0/data/info.pb
02:10:47.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.63.1
02:10:47.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:10:47.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.30.63.62:35377,127.30.63.61:39749,127.30.63.60:46699
02:10:47.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:10:47.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:10:47.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:10:47.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:10:47.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:10:47.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.30.63.1
02:10:47.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster2513987901329325466/ts-0/logs
02:10:47.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:10:47.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:10:47.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:10:47.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
02:10:47.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:10:47.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 5688a9afc53de46b2b2f6847bc5217f38e926fe8
02:10:47.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:10:47.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 24 Oct 2025 01:43:20 UTC on 5fd53c4cbb9d
02:10:47.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8689
02:10:47.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:10:47.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:47.374964 31256 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:10:47.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:47.377326 31256 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:10:47.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:47.404000 31262 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:10:47.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:47.566921 31252 catalog_manager.cc:1269] Loaded cluster ID: 2594582366af42bfa6861b383a2b4ad5
02:10:47.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:47.567239 31252 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 72177680c3ff4418ada40fd5f7ef31da: loading cluster ID for follower catalog manager: success
02:10:47.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:47.575208 31252 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 72177680c3ff4418ada40fd5f7ef31da: acquiring CA information for follower catalog manager: success
02:10:47.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:47.582059 31252 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 72177680c3ff4418ada40fd5f7ef31da: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
02:10:47.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:47.726644 31127 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.63.60" port: 46699 } attempt: 8
02:10:47.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:47.881676 31127 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 47c8574df61247388b26dff25f225818: Bootstrap starting.
02:10:48.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:48.002266 31127 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 47c8574df61247388b26dff25f225818: Neither blocks nor log segments found. Creating new log.
02:10:48.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:48.035355 31127 log.cc:826] T 00000000000000000000000000000000 P 47c8574df61247388b26dff25f225818: Log is configured to *not* fsync() on all Append() calls
02:10:48.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:48.090950 31127 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 47c8574df61247388b26dff25f225818: No bootstrap required, opened a new log
02:10:48.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:48.189028 31156 proxy.cc:239] Call had error, refreshing address and retrying: Timed out: RequestConsensusVote RPC to 127.30.63.61:39749 timed out after 1.970s (SENT)
02:10:48.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:48.332472 31127 raft_consensus.cc:359] T 00000000000000000000000000000000 P 47c8574df61247388b26dff25f225818 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "72177680c3ff4418ada40fd5f7ef31da" member_type: VOTER last_known_addr { host: "127.30.63.62" port: 35377 } } peers { permanent_uuid: "47c8574df61247388b26dff25f225818" member_type: VOTER last_known_addr { host: "127.30.63.61" port: 39749 } } peers { permanent_uuid: "4672000d27184e65a1eb4f4a11332b56" member_type: VOTER last_known_addr { host: "127.30.63.60" port: 46699 } }
02:10:48.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:48.336565 31127 raft_consensus.cc:385] T 00000000000000000000000000000000 P 47c8574df61247388b26dff25f225818 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:10:48.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:48.336931 31127 raft_consensus.cc:740] T 00000000000000000000000000000000 P 47c8574df61247388b26dff25f225818 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 47c8574df61247388b26dff25f225818, State: Initialized, Role: FOLLOWER
02:10:48.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:48.340037 31127 consensus_queue.cc:260] T 00000000000000000000000000000000 P 47c8574df61247388b26dff25f225818 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "72177680c3ff4418ada40fd5f7ef31da" member_type: VOTER last_known_addr { host: "127.30.63.62" port: 35377 } } peers { permanent_uuid: "47c8574df61247388b26dff25f225818" member_type: VOTER last_known_addr { host: "127.30.63.61" port: 39749 } } peers { permanent_uuid: "4672000d27184e65a1eb4f4a11332b56" member_type: VOTER last_known_addr { host: "127.30.63.60" port: 46699 } }
02:10:48.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:48.352224 31272 sys_catalog.cc:455] T 00000000000000000000000000000000 P 47c8574df61247388b26dff25f225818 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "72177680c3ff4418ada40fd5f7ef31da" member_type: VOTER last_known_addr { host: "127.30.63.62" port: 35377 } } peers { permanent_uuid: "47c8574df61247388b26dff25f225818" member_type: VOTER last_known_addr { host: "127.30.63.61" port: 39749 } } peers { permanent_uuid: "4672000d27184e65a1eb4f4a11332b56" member_type: VOTER last_known_addr { host: "127.30.63.60" port: 46699 } } }
02:10:48.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:48.353282 31272 sys_catalog.cc:458] T 00000000000000000000000000000000 P 47c8574df61247388b26dff25f225818 [sys.catalog]: This master's current role is: FOLLOWER
02:10:48.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:48.355255 31127 sys_catalog.cc:565] T 00000000000000000000000000000000 P 47c8574df61247388b26dff25f225818 [sys.catalog]: configured and running, proceeding with master startup.
02:10:48.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:48.355437 31102 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "4672000d27184e65a1eb4f4a11332b56" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "47c8574df61247388b26dff25f225818" is_pre_election: true
02:10:48.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:48.356384 31102 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 47c8574df61247388b26dff25f225818 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 4672000d27184e65a1eb4f4a11332b56 in term 0.
02:10:48.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:48.356986 31102 rpcz_store.cc:267] Call kudu.consensus.ConsensusService.RequestConsensusVote from 127.0.0.1:33190 (request call id 1) took 2138 ms (client timeout 1970 ms). Trace:
02:10:48.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:48.357395 31102 rpcz_store.cc:269] 1024 02:10:46.218540 (+ 0us) service_pool.cc:168] Inserting onto call queue
02:10:48.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1024 02:10:46.218694 (+ 154us) service_pool.cc:225] Handling call
02:10:48.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1024 02:10:48.356896 (+2138202us) inbound_call.cc:173] Queueing success response
02:10:48.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
02:10:48.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:48.359941 31101 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "4672000d27184e65a1eb4f4a11332b56" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "47c8574df61247388b26dff25f225818"
02:10:48.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:48.360598 31101 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 47c8574df61247388b26dff25f225818 [term 0 FOLLOWER]: Advancing to term 1
02:10:48.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:48.366706 31099 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "4672000d27184e65a1eb4f4a11332b56" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "47c8574df61247388b26dff25f225818" is_pre_election: true
02:10:48.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:48.369706 31101 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 47c8574df61247388b26dff25f225818 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 4672000d27184e65a1eb4f4a11332b56 in term 1.
02:10:48.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:48.370911 31100 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 47c8574df61247388b26dff25f225818 [term 1 FOLLOWER]: Refusing update from remote peer 4672000d27184e65a1eb4f4a11332b56: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
02:10:48.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:48.376691 31100 rpcz_store.cc:275] Call kudu.consensus.ConsensusService.UpdateConsensus from 127.0.0.1:33190 (request call id 3) took 1969 ms. Trace:
02:10:48.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:48.377048 31100 rpcz_store.cc:276] 1024 02:10:46.406882 (+ 0us) service_pool.cc:168] Inserting onto call queue
02:10:48.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1024 02:10:46.407063 (+ 181us) service_pool.cc:225] Handling call
02:10:48.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1024 02:10:48.370321 (+1963258us) raft_consensus.cc:1471] Updating replica for 0 ops
02:10:48.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1024 02:10:48.371253 (+ 932us) raft_consensus.cc:1697] Filling consensus response to leader.
02:10:48.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1024 02:10:48.376400 (+ 5147us) spinlock_profiling.cc:243] Waited 2.87 ms on lock 0x7b5400030550. stack: 00007fab1cc86463 00007fab1cc86360 00007fab1c7fc574 0000000000506824 00000000005067b9 00007fab23ccd4c1 00007fab23d6344d 00007fab24d929d9 00007fab23444d34 00007fab23444cc2 00007fab23444bf1 00007fab23444b6c 00007fab23443e22 00007fab21a5eecc 00007fab21a5e306 00007fab21a5dcaf
02:10:48.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1024 02:10:48.376626 (+ 226us) inbound_call.cc:173] Queueing success response
02:10:48.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"spinlock_wait_cycles":6593408}
02:10:48.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:48.378355 31101 rpcz_store.cc:267] Call kudu.consensus.ConsensusService.RequestConsensusVote from 127.0.0.1:33190 (request call id 2) took 2098 ms (client timeout 2242 ms). Trace:
02:10:48.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:48.378733 31101 rpcz_store.cc:269] 1024 02:10:46.279781 (+ 0us) service_pool.cc:168] Inserting onto call queue
02:10:48.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1024 02:10:46.279961 (+ 180us) service_pool.cc:225] Handling call
02:10:48.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1024 02:10:48.378293 (+2098332us) inbound_call.cc:173] Queueing success response
02:10:48.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
02:10:48.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:48.379285 31271 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 4672000d27184e65a1eb4f4a11332b56 [LEADER]: Connected to new peer: Peer: permanent_uuid: "47c8574df61247388b26dff25f225818" member_type: VOTER last_known_addr { host: "127.30.63.61" port: 39749 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
02:10:48.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:48.434780 31272 sys_catalog.cc:455] T 00000000000000000000000000000000 P 47c8574df61247388b26dff25f225818 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 4672000d27184e65a1eb4f4a11332b56. Latest consensus state: current_term: 1 leader_uuid: "4672000d27184e65a1eb4f4a11332b56" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "72177680c3ff4418ada40fd5f7ef31da" member_type: VOTER last_known_addr { host: "127.30.63.62" port: 35377 } } peers { permanent_uuid: "47c8574df61247388b26dff25f225818" member_type: VOTER last_known_addr { host: "127.30.63.61" port: 39749 } } peers { permanent_uuid: "4672000d27184e65a1eb4f4a11332b56" member_type: VOTER last_known_addr { host: "127.30.63.60" port: 46699 } } }
02:10:48.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:48.435017 31280 sys_catalog.cc:455] T 00000000000000000000000000000000 P 47c8574df61247388b26dff25f225818 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "4672000d27184e65a1eb4f4a11332b56" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "72177680c3ff4418ada40fd5f7ef31da" member_type: VOTER last_known_addr { host: "127.30.63.62" port: 35377 } } peers { permanent_uuid: "47c8574df61247388b26dff25f225818" member_type: VOTER last_known_addr { host: "127.30.63.61" port: 39749 } } peers { permanent_uuid: "4672000d27184e65a1eb4f4a11332b56" member_type: VOTER last_known_addr { host: "127.30.63.60" port: 46699 } } }
02:10:48.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:48.435554 31272 sys_catalog.cc:458] T 00000000000000000000000000000000 P 47c8574df61247388b26dff25f225818 [sys.catalog]: This master's current role is: FOLLOWER
02:10:48.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:48.435832 31280 sys_catalog.cc:458] T 00000000000000000000000000000000 P 47c8574df61247388b26dff25f225818 [sys.catalog]: This master's current role is: FOLLOWER
02:10:48.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:48.476436 31288 catalog_manager.cc:805] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
02:10:48.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:48.874624 31271 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4672000d27184e65a1eb4f4a11332b56 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "4672000d27184e65a1eb4f4a11332b56" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "72177680c3ff4418ada40fd5f7ef31da" member_type: VOTER last_known_addr { host: "127.30.63.62" port: 35377 } } peers { permanent_uuid: "47c8574df61247388b26dff25f225818" member_type: VOTER last_known_addr { host: "127.30.63.61" port: 39749 } } peers { permanent_uuid: "4672000d27184e65a1eb4f4a11332b56" member_type: VOTER last_known_addr { host: "127.30.63.60" port: 46699 } } }
02:10:48.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:48.875401 31271 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4672000d27184e65a1eb4f4a11332b56 [sys.catalog]: This master's current role is: LEADER
02:10:49.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:49.510861 31288 catalog_manager.cc:1269] Loaded cluster ID: 2594582366af42bfa6861b383a2b4ad5
02:10:49.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:49.513005 31288 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 47c8574df61247388b26dff25f225818: loading cluster ID for follower catalog manager: success
02:10:49.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:49.520538 31288 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 47c8574df61247388b26dff25f225818: acquiring CA information for follower catalog manager: success
02:10:49.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:49.554811 31288 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 47c8574df61247388b26dff25f225818: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
02:10:50.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:48.797922 31261 debug-util.cc:398] Leaking SignalData structure 0x7b0800037500 after lost signal to thread 31256
02:10:50.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:47.408635 31263 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:10:50.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:50.160264 31261 kernel_stack_watchdog.cc:198] Thread 31256 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 393ms:
02:10:50.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
02:10:50.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
02:10:50.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
02:10:50.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
02:10:50.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:50.161520 31256 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 2.759s user 0.993s sys 1.570s
02:10:50.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:50.161877 31256 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 2.759s user 0.993s sys 1.570s
02:10:50.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:50.173344 31295 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:10:50.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:50.176400 31256 server_base.cc:1047] running on GCE node
02:10:50.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:50.178102 31256 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:10:50.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:50.181386 31256 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:10:50.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:50.183110 31256 hybrid_clock.cc:648] HybridClock initialized: now 1761271850183062 us; error 51 us; skew 500 ppm
02:10:50.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:50.184599 31256 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:10:50.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:50.193148 31256 webserver.cc:492] Webserver started at http://127.30.63.1:43961/ using document root <none> and password file <none>
02:10:50.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:50.194748 31256 fs_manager.cc:362] Metadata directory not provided
02:10:50.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:50.195789 31256 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:10:50.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:50.196550 31256 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
02:10:50.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:50.204058 31256 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster2513987901329325466/ts-0/data/instance:
02:10:50.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ff2940a8e85e40ecb4c8e4a363933a7d"
02:10:50.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-24 02:10:50 on dist-test-slave-xv5f"
02:10:50.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:50.205965 31256 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster2513987901329325466/ts-0/wal/instance:
02:10:50.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ff2940a8e85e40ecb4c8e4a363933a7d"
02:10:50.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-24 02:10:50 on dist-test-slave-xv5f"
02:10:50.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:50.217087 31256 fs_manager.cc:696] Time spent creating directory manager: real 0.010s user 0.002s sys 0.007s
02:10:50.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:50.229351 31300 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:10:50.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:50.231453 31256 fs_manager.cc:730] Time spent opening block manager: real 0.009s user 0.001s sys 0.004s
02:10:50.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:50.231904 31256 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster2513987901329325466/ts-0/data,/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster2513987901329325466/ts-0/wal
02:10:50.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ff2940a8e85e40ecb4c8e4a363933a7d"
02:10:50.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-24 02:10:50 on dist-test-slave-xv5f"
02:10:50.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:50.232367 31256 fs_report.cc:389] FS layout report
02:10:50.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:10:50.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster2513987901329325466/ts-0/wal
02:10:50.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster2513987901329325466/ts-0/wal
02:10:50.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster2513987901329325466/ts-0/data/data
02:10:50.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:10:50.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:10:50.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:10:50.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:10:50.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:10:50.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:10:50.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:10:50.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:10:50.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:10:50.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:10:50.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:10:50.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:50.319541 31256 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:10:50.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:50.321098 31256 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:10:50.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:50.321599 31256 kserver.cc:163] Server-wide thread pool size limit: 3276
02:10:50.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:50.325116 31256 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
02:10:50.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:50.329659 31256 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
02:10:50.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:50.329900 31256 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
02:10:50.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:50.330221 31256 ts_tablet_manager.cc:616] Registered 0 tablets
02:10:50.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:50.330411 31256 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
02:10:50.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:50.513324 31256 rpc_server.cc:307] RPC server started. Bound to: 127.30.63.1:38817
02:10:50.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:50.513440 31412 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.30.63.1:38817 every 8 connection(s)
02:10:50.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:50.516196 31256 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster2513987901329325466/ts-0/data/info.pb
02:10:50.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:50.517192 30972 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskOg0PFu/build/tsan/bin/kudu as pid 31256
02:10:50.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:50.517942 30972 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster2513987901329325466/ts-0/wal/instance
02:10:50.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:50.547289 30972 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskOg0PFu/build/tsan/bin/kudu
02:10:50.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskOg0PFu/build/tsan/bin/kudu
02:10:50.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster2513987901329325466/ts-1/wal
02:10:50.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster2513987901329325466/ts-1/data
02:10:50.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:10:50.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:10:50.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:10:50.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:10:50.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:10:50.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:10:50.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster2513987901329325466/ts-1/logs
02:10:50.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster2513987901329325466/ts-1/data/info.pb
02:10:50.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:10:50.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:10:50.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:10:50.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:10:50.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:10:50.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:10:50.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:10:50.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:10:50.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
02:10:50.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:10:50.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.63.2:0
02:10:50.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.30.63.2
02:10:50.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.63.2
02:10:50.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:10:50.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.30.63.62:35377,127.30.63.61:39749,127.30.63.60:46699
02:10:50.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.63.20:35311
02:10:50.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:10:50.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:10:50.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
02:10:50.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:50.570495 31413 heartbeater.cc:344] Connected to a master server at 127.30.63.60:46699
02:10:50.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:50.571105 31413 heartbeater.cc:461] Registering TS with master...
02:10:50.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:50.572513 31413 heartbeater.cc:507] Master 127.30.63.60:46699 requested a full tablet report, sending...
02:10:50.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:50.576468 31174 ts_manager.cc:194] Registered new tserver with Master: ff2940a8e85e40ecb4c8e4a363933a7d (127.30.63.1:38817)
02:10:50.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:50.580905 31174 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.30.63.1:42789
02:10:50.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:50.610833 31417 heartbeater.cc:344] Connected to a master server at 127.30.63.62:35377
02:10:50.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:50.611258 31417 heartbeater.cc:461] Registering TS with master...
02:10:50.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:50.611995 31417 heartbeater.cc:507] Master 127.30.63.62:35377 requested a full tablet report, sending...
02:10:50.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:50.615198 31414 heartbeater.cc:344] Connected to a master server at 127.30.63.61:39749
02:10:50.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:50.615258 31016 ts_manager.cc:194] Registered new tserver with Master: ff2940a8e85e40ecb4c8e4a363933a7d (127.30.63.1:38817)
02:10:50.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:50.615597 31414 heartbeater.cc:461] Registering TS with master...
02:10:50.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:50.616282 31414 heartbeater.cc:507] Master 127.30.63.61:39749 requested a full tablet report, sending...
02:10:50.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:50.619256 31092 ts_manager.cc:194] Registered new tserver with Master: ff2940a8e85e40ecb4c8e4a363933a7d (127.30.63.1:38817)
02:10:50.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:50.927266 31418 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:10:50.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:50.927742 31418 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:10:50.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:50.928195 31418 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:10:50.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:50.962476 31418 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:10:50.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:50.963382 31418 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.30.63.2
02:10:51.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:51.005024 31418 tablet_server_runner.cc:78] Tablet server non-default flags:
02:10:51.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:10:51.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.63.20:35311
02:10:51.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:10:51.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:10:51.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster2513987901329325466/ts-1/data
02:10:51.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster2513987901329325466/ts-1/wal
02:10:51.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:10:51.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:10:51.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.63.2:0
02:10:51.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:10:51.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:10:51.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:10:51.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster2513987901329325466/ts-1/data/info.pb
02:10:51.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.63.2
02:10:51.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:10:51.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.30.63.62:35377,127.30.63.61:39749,127.30.63.60:46699
02:10:51.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:10:51.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:10:51.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:10:51.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:10:51.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:10:51.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.30.63.2
02:10:51.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster2513987901329325466/ts-1/logs
02:10:51.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:10:51.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:10:51.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:10:51.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
02:10:51.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:10:51.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 5688a9afc53de46b2b2f6847bc5217f38e926fe8
02:10:51.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:10:51.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 24 Oct 2025 01:43:20 UTC on 5fd53c4cbb9d
02:10:51.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8689
02:10:51.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:10:51.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:51.006343 31418 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:10:51.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:51.008008 31418 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:10:51.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:51.024350 31429 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:10:51.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:51.589371 31413 heartbeater.cc:499] Master 127.30.63.60:46699 was elected leader, sending a full tablet report...
02:10:52.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:52.424144 31427 debug-util.cc:398] Leaking SignalData structure 0x7b0800037500 after lost signal to thread 31418
02:10:52.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:52.819980 31427 kernel_stack_watchdog.cc:198] Thread 31418 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 396ms:
02:10:52.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
02:10:52.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
02:10:52.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
02:10:52.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
02:10:52.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:52.820858 31418 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.795s user 0.550s sys 1.042s
02:10:52.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:52.821256 31418 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.796s user 0.550s sys 1.042s
02:10:52.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:52.821753 31430 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection timed out after 1796 milliseconds
02:10:52.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:51.026733 31428 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:10:52.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:52.823180 31431 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:10:52.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:52.823232 31418 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:10:52.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:52.827474 31418 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:10:52.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:52.830163 31418 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:10:52.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:52.831594 31418 hybrid_clock.cc:648] HybridClock initialized: now 1761271852831556 us; error 77 us; skew 500 ppm
02:10:52.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:52.832664 31418 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:10:52.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:52.839352 31418 webserver.cc:492] Webserver started at http://127.30.63.2:45763/ using document root <none> and password file <none>
02:10:52.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:52.840332 31418 fs_manager.cc:362] Metadata directory not provided
02:10:52.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:52.840523 31418 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:10:52.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:52.840950 31418 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
02:10:52.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:52.845485 31418 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster2513987901329325466/ts-1/data/instance:
02:10:52.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6625dd12105b401c9c12e10f78ed1c84"
02:10:52.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-24 02:10:52 on dist-test-slave-xv5f"
02:10:52.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:52.846827 31418 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster2513987901329325466/ts-1/wal/instance:
02:10:52.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6625dd12105b401c9c12e10f78ed1c84"
02:10:52.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-24 02:10:52 on dist-test-slave-xv5f"
02:10:52.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:52.855050 31418 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.005s sys 0.002s
02:10:52.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:52.862600 31443 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:10:52.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:52.863919 31418 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.001s sys 0.002s
02:10:52.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:52.864333 31418 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster2513987901329325466/ts-1/data,/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster2513987901329325466/ts-1/wal
02:10:52.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6625dd12105b401c9c12e10f78ed1c84"
02:10:52.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-24 02:10:52 on dist-test-slave-xv5f"
02:10:52.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:52.864660 31418 fs_report.cc:389] FS layout report
02:10:52.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:10:52.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster2513987901329325466/ts-1/wal
02:10:52.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster2513987901329325466/ts-1/wal
02:10:52.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster2513987901329325466/ts-1/data/data
02:10:52.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:10:52.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:10:52.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:10:52.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:10:52.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:10:52.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:10:52.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:10:52.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:10:52.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:10:52.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:10:52.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:10:52.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:52.923306 31418 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:10:52.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:52.924842 31418 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:10:52.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:52.925283 31418 kserver.cc:163] Server-wide thread pool size limit: 3276
02:10:52.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:52.928886 31418 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
02:10:52.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:52.933480 31418 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
02:10:52.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:52.933706 31418 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
02:10:52.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:52.933954 31418 ts_tablet_manager.cc:616] Registered 0 tablets
02:10:52.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:52.934100 31418 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
02:10:53.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:53.078881 31418 rpc_server.cc:307] RPC server started. Bound to: 127.30.63.2:46827
02:10:53.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:53.078989 31555 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.30.63.2:46827 every 8 connection(s)
02:10:53.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:53.081419 31418 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster2513987901329325466/ts-1/data/info.pb
02:10:53.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:53.090757 30972 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskOg0PFu/build/tsan/bin/kudu as pid 31418
02:10:53.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:53.091288 30972 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster2513987901329325466/ts-1/wal/instance
02:10:53.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:53.110613 30972 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskOg0PFu/build/tsan/bin/kudu
02:10:53.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskOg0PFu/build/tsan/bin/kudu
02:10:53.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster2513987901329325466/ts-2/wal
02:10:53.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster2513987901329325466/ts-2/data
02:10:53.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:10:53.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:10:53.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:10:53.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:10:53.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:10:53.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:10:53.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster2513987901329325466/ts-2/logs
02:10:53.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster2513987901329325466/ts-2/data/info.pb
02:10:53.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:10:53.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:10:53.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:10:53.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:10:53.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:10:53.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:10:53.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:10:53.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:10:53.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
02:10:53.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:10:53.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.63.3:0
02:10:53.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.30.63.3
02:10:53.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.63.3
02:10:53.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:10:53.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.30.63.62:35377,127.30.63.61:39749,127.30.63.60:46699
02:10:53.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.63.20:35311
02:10:53.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:10:53.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:10:53.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
02:10:53.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:53.127681 31557 heartbeater.cc:344] Connected to a master server at 127.30.63.61:39749
02:10:53.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:53.128232 31557 heartbeater.cc:461] Registering TS with master...
02:10:53.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:53.129711 31557 heartbeater.cc:507] Master 127.30.63.61:39749 requested a full tablet report, sending...
02:10:53.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:53.129951 31558 heartbeater.cc:344] Connected to a master server at 127.30.63.62:35377
02:10:53.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:53.130295 31558 heartbeater.cc:461] Registering TS with master...
02:10:53.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:53.131325 31558 heartbeater.cc:507] Master 127.30.63.62:35377 requested a full tablet report, sending...
02:10:53.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:53.133713 31556 heartbeater.cc:344] Connected to a master server at 127.30.63.60:46699
02:10:53.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:53.134099 31556 heartbeater.cc:461] Registering TS with master...
02:10:53.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:53.134068 31016 ts_manager.cc:194] Registered new tserver with Master: 6625dd12105b401c9c12e10f78ed1c84 (127.30.63.2:46827)
02:10:53.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:53.134546 31092 ts_manager.cc:194] Registered new tserver with Master: 6625dd12105b401c9c12e10f78ed1c84 (127.30.63.2:46827)
02:10:53.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:53.135171 31556 heartbeater.cc:507] Master 127.30.63.60:46699 requested a full tablet report, sending...
02:10:53.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:53.137280 31174 ts_manager.cc:194] Registered new tserver with Master: 6625dd12105b401c9c12e10f78ed1c84 (127.30.63.2:46827)
02:10:53.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:53.138615 31174 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.30.63.2:56225
02:10:53.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:53.430917 31566 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:10:53.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:53.431468 31566 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:10:53.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:53.432000 31566 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:10:53.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:53.465155 31566 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:10:53.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:53.466116 31566 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.30.63.3
02:10:53.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:53.503486 31566 tablet_server_runner.cc:78] Tablet server non-default flags:
02:10:53.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:10:53.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.63.20:35311
02:10:53.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:10:53.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:10:53.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster2513987901329325466/ts-2/data
02:10:53.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster2513987901329325466/ts-2/wal
02:10:53.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:10:53.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:10:53.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.63.3:0
02:10:53.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:10:53.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:10:53.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:10:53.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster2513987901329325466/ts-2/data/info.pb
02:10:53.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.63.3
02:10:53.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:10:53.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.30.63.62:35377,127.30.63.61:39749,127.30.63.60:46699
02:10:53.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:10:53.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:10:53.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:10:53.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:10:53.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:10:53.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.30.63.3
02:10:53.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster2513987901329325466/ts-2/logs
02:10:53.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:10:53.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:10:53.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:10:53.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
02:10:53.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:10:53.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 5688a9afc53de46b2b2f6847bc5217f38e926fe8
02:10:53.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:10:53.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 24 Oct 2025 01:43:20 UTC on 5fd53c4cbb9d
02:10:53.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8689
02:10:53.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:10:53.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:53.505059 31566 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:10:53.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:53.506665 31566 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:10:53.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:53.521656 31572 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:10:54.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:54.141691 31556 heartbeater.cc:499] Master 127.30.63.60:46699 was elected leader, sending a full tablet report...
02:10:54.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:53.527624 31575 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:10:54.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:53.523902 31573 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:10:54.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:54.854866 31574 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1326 milliseconds
02:10:54.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:54.855005 31566 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:10:54.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:54.856159 31566 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:10:54.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:54.859009 31566 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:10:54.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:54.860483 31566 hybrid_clock.cc:648] HybridClock initialized: now 1761271854860425 us; error 68 us; skew 500 ppm
02:10:54.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:54.861316 31566 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:10:54.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:54.868009 31566 webserver.cc:492] Webserver started at http://127.30.63.3:43669/ using document root <none> and password file <none>
02:10:54.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:54.868974 31566 fs_manager.cc:362] Metadata directory not provided
02:10:54.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:54.869164 31566 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:10:54.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:54.869565 31566 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
02:10:54.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:54.874058 31566 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster2513987901329325466/ts-2/data/instance:
02:10:54.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bafbedc6c71947ceb4fb063ee390cf9f"
02:10:54.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-24 02:10:54 on dist-test-slave-xv5f"
02:10:54.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:54.875536 31566 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster2513987901329325466/ts-2/wal/instance:
02:10:54.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bafbedc6c71947ceb4fb063ee390cf9f"
02:10:54.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-24 02:10:54 on dist-test-slave-xv5f"
02:10:54.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:54.883111 31566 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.006s sys 0.001s
02:10:54.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:54.889497 31583 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:10:54.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:54.890784 31566 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.001s
02:10:54.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:54.891182 31566 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster2513987901329325466/ts-2/data,/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster2513987901329325466/ts-2/wal
02:10:54.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bafbedc6c71947ceb4fb063ee390cf9f"
02:10:54.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-24 02:10:54 on dist-test-slave-xv5f"
02:10:54.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:54.891535 31566 fs_report.cc:389] FS layout report
02:10:54.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:10:54.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster2513987901329325466/ts-2/wal
02:10:54.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster2513987901329325466/ts-2/wal
02:10:54.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster2513987901329325466/ts-2/data/data
02:10:54.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:10:54.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:10:54.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:10:54.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:10:54.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:10:54.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:10:54.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:10:54.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:10:54.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:10:54.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:10:54.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:10:54.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:54.939440 31566 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:10:54.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:54.940994 31566 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:10:54.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:54.941515 31566 kserver.cc:163] Server-wide thread pool size limit: 3276
02:10:54.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:54.944202 31566 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
02:10:54.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:54.948674 31566 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
02:10:54.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:54.948920 31566 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
02:10:54.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:54.949229 31566 ts_tablet_manager.cc:616] Registered 0 tablets
02:10:54.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:54.949391 31566 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
02:10:55.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:55.093250 31566 rpc_server.cc:307] RPC server started. Bound to: 127.30.63.3:36131
02:10:55.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:55.093381 31695 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.30.63.3:36131 every 8 connection(s)
02:10:55.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:55.096892 31566 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster2513987901329325466/ts-2/data/info.pb
02:10:55.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:55.106201 30972 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskOg0PFu/build/tsan/bin/kudu as pid 31566
02:10:55.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:55.106830 30972 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster2513987901329325466/ts-2/wal/instance
02:10:55.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:55.141404 31697 heartbeater.cc:344] Connected to a master server at 127.30.63.61:39749
02:10:55.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:55.142002 31697 heartbeater.cc:461] Registering TS with master...
02:10:55.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:55.143429 31697 heartbeater.cc:507] Master 127.30.63.61:39749 requested a full tablet report, sending...
02:10:55.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:55.147126 31698 heartbeater.cc:344] Connected to a master server at 127.30.63.62:35377
02:10:55.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:55.147544 31698 heartbeater.cc:461] Registering TS with master...
02:10:55.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:55.147605 31091 ts_manager.cc:194] Registered new tserver with Master: bafbedc6c71947ceb4fb063ee390cf9f (127.30.63.3:36131)
02:10:55.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:55.148525 31698 heartbeater.cc:507] Master 127.30.63.62:35377 requested a full tablet report, sending...
02:10:55.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:55.152473 31016 ts_manager.cc:194] Registered new tserver with Master: bafbedc6c71947ceb4fb063ee390cf9f (127.30.63.3:36131)
02:10:55.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:55.155002 31696 heartbeater.cc:344] Connected to a master server at 127.30.63.60:46699
02:10:55.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:55.155398 31696 heartbeater.cc:461] Registering TS with master...
02:10:55.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:55.156278 31696 heartbeater.cc:507] Master 127.30.63.60:46699 requested a full tablet report, sending...
02:10:55.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:55.158366 31174 ts_manager.cc:194] Registered new tserver with Master: bafbedc6c71947ceb4fb063ee390cf9f (127.30.63.3:36131)
02:10:55.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:55.160112 31174 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.30.63.3:48731
02:10:55.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:55.174017 30972 external_mini_cluster.cc:949] 3 TS(s) registered with all masters
02:10:55.185 [DEBUG - main] (MiniKuduCluster.java:195) Response:
02:10:55.185 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
02:10:55.186 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.30.63.62"
port: 35377
}
bound_http_address {
host: "127.30.63.62"
port: 42619
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.30.63.61"
port: 39749
}
bound_http_address {
host: "127.30.63.61"
port: 44831
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.30.63.60"
port: 46699
}
bound_http_address {
host: "127.30.63.60"
port: 34169
}
}
}
02:10:55.186 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
02:10:55.187 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.30.63.1"
port: 38817
}
bound_http_address {
host: "127.30.63.1"
port: 43961
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.30.63.2"
port: 46827
}
bound_http_address {
host: "127.30.63.2"
port: 45763
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.30.63.3"
port: 36131
}
bound_http_address {
host: "127.30.63.3"
port: 43669
}
}
}
02:10:55.187 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
02:10:55.188 [DEBUG - main] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3368569904505815558/krb5kdc/krb5cc
02:10:55.188 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
02:10:55.189 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskOg0PFu/test-tmp
02:10:55.190 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
02:10:55.190 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskOg0PFu/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
02:10:55.191 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656"
extra_master_flags: "--webserver_require_spnego=false"
extra_master_flags: "--authn_token_validity_seconds=10"
extra_tserver_flags: "--webserver_require_spnego=false"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
02:10:55.536 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:10:55.537 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
02:10:55.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:55.540096 31707 mini_oidc.cc:150] Starting JWKS server
02:10:55.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:55.543256 31707 webserver.cc:492] Webserver started at http://127.0.0.1:42087/ using document root <none> and password file <none>
02:10:55.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:55.543658 31707 mini_oidc.cc:186] Starting OIDC Discovery server
02:10:55.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:55.545313 31707 webserver.cc:492] Webserver started at http://127.0.0.1:38717/ using document root <none> and password file <none>
02:10:55.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:55.564567 31048 debug-util.cc:398] Leaking SignalData structure 0x7b080009b220 after lost signal to thread 30985
02:10:55.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
02:10:55.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/krb5kdc/principal' for realm 'KRBTEST.COM',
02:10:55.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
02:10:55.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 24 02:10:55 dist-test-slave-xv5f krb5kdc[31719](info): setting up network...
02:10:55.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
02:10:55.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 24 02:10:55 dist-test-slave-xv5f krb5kdc[31719](info): set up 2 sockets
02:10:55.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 24 02:10:55 dist-test-slave-xv5f krb5kdc[31719](info): commencing operation
02:10:55.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
02:10:56.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:56.163175 31696 heartbeater.cc:499] Master 127.30.63.60:46699 was elected leader, sending a full tablet report...
02:10:57.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:57.678220 31707 mini_kdc.cc:121] Time spent starting KDC: real 2.095s user 0.004s sys 0.015s
02:10:57.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
02:10:57.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
02:10:57.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
02:10:57.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
02:10:57.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
02:10:57.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
02:10:57.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
02:10:57.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
02:10:57.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
02:10:57.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
02:10:57.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/krb5kdc/test-user.keytab.
02:10:57.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/krb5kdc/test-user.keytab.
02:10:57.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 24 02:10:57 dist-test-slave-xv5f krb5kdc[31719](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761271857, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:10:57.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
02:10:57.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-10-24T02:10:57Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
02:10:57.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-10-24T02:10:57Z Disabled control of system clock
02:10:57.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.30.246.254@KRBTEST.COM; defaulting to no policy
02:10:57.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:10:57.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.30.246.254@KRBTEST.COM" created.
02:10:57.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:10:57.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.30.246.254 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/krb5kdc/kudu.keytab.
02:10:57.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.30.246.254 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/krb5kdc/kudu.keytab.
02:10:57.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.30.246.254@KRBTEST.COM; defaulting to no policy
02:10:57.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:10:57.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.30.246.254@KRBTEST.COM" created.
02:10:57.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:10:57.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.30.246.254 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/krb5kdc/kudu.keytab.
02:10:57.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.30.246.254 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/krb5kdc/kudu.keytab.
02:10:57.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:57.911336 31707 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskOg0PFu/build/tsan/bin/kudu
02:10:57.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskOg0PFu/build/tsan/bin/kudu
02:10:57.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-0/wal
02:10:57.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-0/data
02:10:57.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:10:57.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:10:57.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:10:57.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:10:57.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:10:57.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:10:57.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-0/logs
02:10:57.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-0/data/info.pb
02:10:57.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:10:57.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:10:57.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:10:57.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:10:57.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:10:57.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:10:57.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:10:57.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:10:57.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:10:57.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:10:57.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:10:57.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:10:57.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.246.254:40109
02:10:57.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.246.254
02:10:57.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:10:57.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/krb5kdc/kudu.keytab
02:10:57.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.30.246.254
02:10:57.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:10:57.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
02:10:57.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
02:10:57.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
02:10:57.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.246.212:35299
02:10:57.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:10:57.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:10:57.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:10:57.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:10:57.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.30.246.254:40109,127.30.246.253:36093,127.30.246.252:39647
02:10:57.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:10:57.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42087/jwks.json
02:10:57.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
02:10:57.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
02:10:58.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:58.221036 31736 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:10:58.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:58.221690 31736 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:10:58.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:58.222469 31736 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:10:58.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:58.254551 31736 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
02:10:58.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:58.254810 31736 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
02:10:58.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:58.255223 31736 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:10:58.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:58.255525 31736 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
02:10:58.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:58.255800 31736 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
02:10:58.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:58.256071 31736 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
02:10:58.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:58.256294 31736 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:42087/jwks.json
02:10:58.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:58.294929 31736 master_runner.cc:386] Master server non-default flags:
02:10:58.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:10:58.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.246.212:35299
02:10:58.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:10:58.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:10:58.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-0/data
02:10:58.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-0/wal
02:10:58.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
02:10:58.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:10:58.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.30.246.254:40109,127.30.246.253:36093,127.30.246.252:39647
02:10:58.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/krb5kdc/kudu.keytab
02:10:58.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.30.246.254
02:10:58.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:10:58.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:10:58.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:10:58.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.246.254:40109
02:10:58.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:10:58.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:10:58.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:10:58.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42087/jwks.json
02:10:58.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:10:58.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
02:10:58.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
02:10:58.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:10:58.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:10:58.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-0/data/info.pb
02:10:58.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.246.254
02:10:58.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:10:58.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:10:58.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:10:58.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:10:58.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:10:58.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:10:58.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-0/logs
02:10:58.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:10:58.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:10:58.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:10:58.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:10:58.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:10:58.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 5688a9afc53de46b2b2f6847bc5217f38e926fe8
02:10:58.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:10:58.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 24 Oct 2025 01:43:20 UTC on 5fd53c4cbb9d
02:10:58.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8689
02:10:58.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:10:58.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:58.296514 31736 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:10:58.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:10:58.297770 31736 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:10:58.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:58.309003 31742 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:11:00.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:59.713060 31741 debug-util.cc:398] Leaking SignalData structure 0x7b0800027080 after lost signal to thread 31736
02:11:00.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:00.147706 31736 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.838s user 0.669s sys 1.147s
02:11:00.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:00.148137 31736 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.839s user 0.669s sys 1.147s
02:11:00.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:10:58.309365 31743 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:11:00.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:00.150239 31745 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:11:00.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:00.153157 31744 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1839 milliseconds
02:11:00.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:00.153259 31736 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:11:00.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:00.154599 31736 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:11:00.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:00.157089 31736 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:11:00.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:00.158597 31736 hybrid_clock.cc:648] HybridClock initialized: now 1761271860158549 us; error 60 us; skew 500 ppm
02:11:00.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:00.159499 31736 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:11:00.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 24 02:11:00 dist-test-slave-xv5f krb5kdc[31719](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761271860, etypes {rep=17 tkt=17 ses=17}, kudu/127.30.246.254@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:11:00.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:00.165627 31736 init.cc:377] Logged in from keytab as kudu/127.30.246.254@KRBTEST.COM (short username kudu)
02:11:00.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:00.170468 31736 webserver.cc:492] Webserver started at http://127.30.246.254:33217/ using document root <none> and password file <none>
02:11:00.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:00.171454 31736 fs_manager.cc:362] Metadata directory not provided
02:11:00.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:00.171686 31736 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:11:00.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:00.172168 31736 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
02:11:00.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:00.178504 31736 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-0/data/instance:
02:11:00.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3feaddadcc464a6daa4fb7b3dc17612c"
02:11:00.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-24 02:11:00 on dist-test-slave-xv5f"
02:11:00.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:00.180275 31736 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-0/wal/instance:
02:11:00.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3feaddadcc464a6daa4fb7b3dc17612c"
02:11:00.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-24 02:11:00 on dist-test-slave-xv5f"
02:11:00.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:00.189543 31736 fs_manager.cc:696] Time spent creating directory manager: real 0.009s user 0.000s sys 0.009s
02:11:00.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:00.195489 31756 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:11:00.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:00.196656 31736 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.003s
02:11:00.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:00.197043 31736 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-0/data,/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-0/wal
02:11:00.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3feaddadcc464a6daa4fb7b3dc17612c"
02:11:00.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-24 02:11:00 on dist-test-slave-xv5f"
02:11:00.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:00.197407 31736 fs_report.cc:389] FS layout report
02:11:00.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:11:00.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-0/wal
02:11:00.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-0/wal
02:11:00.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-0/data/data
02:11:00.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:11:00.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:11:00.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:11:00.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:11:00.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:11:00.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:11:00.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:11:00.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:11:00.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:11:00.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:11:00.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:11:00.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:00.251210 31736 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:11:00.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:00.261545 31736 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:11:00.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:00.262077 31736 kserver.cc:163] Server-wide thread pool size limit: 3276
02:11:00.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:00.338413 31736 rpc_server.cc:307] RPC server started. Bound to: 127.30.246.254:40109
02:11:00.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:00.338495 31807 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.30.246.254:40109 every 8 connection(s)
02:11:00.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:00.341202 31736 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-0/data/info.pb
02:11:00.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:00.344096 31707 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskOg0PFu/build/tsan/bin/kudu as pid 31736
02:11:00.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:00.344759 31707 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-0/wal/instance
02:11:00.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:00.347919 31808 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
02:11:00.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:00.359730 31808 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.30.246.254" port: 40109 } has no permanent_uuid. Determining permanent_uuid...
02:11:00.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.30.246.253@KRBTEST.COM; defaulting to no policy
02:11:00.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:11:00.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.30.246.253@KRBTEST.COM" created.
02:11:00.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 24 02:11:00 dist-test-slave-xv5f krb5kdc[31719](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761271860, etypes {rep=17 tkt=17 ses=17}, kudu/127.30.246.254@KRBTEST.COM for kudu/127.30.246.254@KRBTEST.COM
02:11:00.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:11:00.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.30.246.253 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/krb5kdc/kudu.keytab.
02:11:00.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.30.246.253 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/krb5kdc/kudu.keytab.
02:11:00.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:00.400683 31808 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.30.246.253" port: 36093 } has no permanent_uuid. Determining permanent_uuid...
02:11:00.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:00.403283 31759 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.30.246.253:36093: connect: Connection refused (error 111)
02:11:00.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.30.246.253@KRBTEST.COM; defaulting to no policy
02:11:00.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:11:00.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.30.246.253@KRBTEST.COM" created.
02:11:00.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:00.407646 31808 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.30.246.253:36093: Network error: Client connection negotiation failed: client connection to 127.30.246.253:36093: connect: Connection refused (error 111)
02:11:00.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:11:00.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.30.246.253 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/krb5kdc/kudu.keytab.
02:11:00.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.30.246.253 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/krb5kdc/kudu.keytab.
02:11:00.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:00.428714 31707 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskOg0PFu/build/tsan/bin/kudu
02:11:00.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskOg0PFu/build/tsan/bin/kudu
02:11:00.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-1/wal
02:11:00.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-1/data
02:11:00.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:11:00.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:11:00.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:11:00.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:11:00.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:11:00.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:11:00.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-1/logs
02:11:00.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-1/data/info.pb
02:11:00.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:11:00.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:11:00.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:11:00.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:11:00.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:11:00.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:11:00.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:11:00.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:11:00.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:11:00.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:11:00.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:11:00.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:11:00.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.246.253:36093
02:11:00.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.246.253
02:11:00.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:11:00.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/krb5kdc/kudu.keytab
02:11:00.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.30.246.253
02:11:00.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:11:00.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
02:11:00.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
02:11:00.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
02:11:00.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.246.212:35299
02:11:00.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:11:00.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:11:00.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:11:00.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:11:00.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.30.246.254:40109,127.30.246.253:36093,127.30.246.252:39647
02:11:00.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:11:00.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42087/jwks.json
02:11:00.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
02:11:00.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
02:11:00.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:00.457226 31808 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.246.253" port: 36093 } attempt: 1
02:11:00.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:00.461169 31808 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.30.246.253:36093: Network error: Client connection negotiation failed: client connection to 127.30.246.253:36093: connect: Connection refused (error 111)
02:11:00.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:00.529671 31808 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.246.253" port: 36093 } attempt: 2
02:11:00.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:00.533615 31808 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.30.246.253:36093: Network error: Client connection negotiation failed: client connection to 127.30.246.253:36093: connect: Connection refused (error 111)
02:11:00.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:00.625170 31808 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.246.253" port: 36093 } attempt: 3
02:11:00.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:00.629300 31808 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.30.246.253:36093: Network error: Client connection negotiation failed: client connection to 127.30.246.253:36093: connect: Connection refused (error 111)
02:11:00.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:00.739534 31817 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:11:00.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:00.740090 31817 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:11:00.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:00.740700 31817 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:11:00.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:00.772837 31808 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.246.253" port: 36093 } attempt: 4
02:11:00.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:00.777199 31808 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.30.246.253:36093: Network error: Client connection negotiation failed: client connection to 127.30.246.253:36093: connect: Connection refused (error 111)
02:11:00.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:00.779314 31817 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
02:11:00.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:00.779541 31817 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
02:11:00.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:00.779809 31817 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:11:00.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:00.780014 31817 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
02:11:00.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:00.780206 31817 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
02:11:00.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:00.780393 31817 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
02:11:00.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:00.780541 31817 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:42087/jwks.json
02:11:00.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:00.819092 31817 master_runner.cc:386] Master server non-default flags:
02:11:00.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:11:00.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.246.212:35299
02:11:00.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:11:00.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:11:00.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-1/data
02:11:00.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-1/wal
02:11:00.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
02:11:00.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:11:00.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.30.246.254:40109,127.30.246.253:36093,127.30.246.252:39647
02:11:00.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/krb5kdc/kudu.keytab
02:11:00.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.30.246.253
02:11:00.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:11:00.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:11:00.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:11:00.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.246.253:36093
02:11:00.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:11:00.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:11:00.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:11:00.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42087/jwks.json
02:11:00.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:11:00.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
02:11:00.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
02:11:00.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:11:00.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:11:00.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-1/data/info.pb
02:11:00.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.246.253
02:11:00.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:11:00.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:11:00.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:11:00.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:11:00.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:11:00.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:11:00.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-1/logs
02:11:00.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:11:00.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:11:00.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:11:00.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:11:00.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:11:00.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 5688a9afc53de46b2b2f6847bc5217f38e926fe8
02:11:00.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:11:00.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 24 Oct 2025 01:43:20 UTC on 5fd53c4cbb9d
02:11:00.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8689
02:11:00.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:11:00.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:00.820385 31817 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:11:00.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:00.821411 31817 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:11:00.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:00.832942 31823 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:11:01.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:01.076637 31808 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.246.253" port: 36093 } attempt: 5
02:11:01.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:01.084990 31808 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.30.246.253:36093: Network error: Client connection negotiation failed: client connection to 127.30.246.253:36093: connect: Connection refused (error 111)
02:11:01.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:01.632602 31808 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.246.253" port: 36093 } attempt: 6
02:11:01.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:01.668169 31808 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.30.246.253:36093: Network error: Client connection negotiation failed: client connection to 127.30.246.253:36093: connect: Connection refused (error 111)
02:11:02.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:02.237239 31822 debug-util.cc:398] Leaking SignalData structure 0x7b0800027080 after lost signal to thread 31817
02:11:02.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:02.349231 31817 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.516s user 0.503s sys 0.961s
02:11:02.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:02.349646 31817 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.516s user 0.503s sys 0.961s
02:11:02.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:00.833726 31824 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:11:02.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:02.351701 31826 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:11:02.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:02.355571 31825 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1517 milliseconds
02:11:02.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:02.355597 31817 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:11:02.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:02.357012 31817 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:11:02.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:02.359450 31817 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:11:02.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:02.360816 31817 hybrid_clock.cc:648] HybridClock initialized: now 1761271862360748 us; error 73 us; skew 500 ppm
02:11:02.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:02.361680 31817 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:11:02.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 24 02:11:02 dist-test-slave-xv5f krb5kdc[31719](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761271862, etypes {rep=17 tkt=17 ses=17}, kudu/127.30.246.253@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:11:02.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:02.367502 31817 init.cc:377] Logged in from keytab as kudu/127.30.246.253@KRBTEST.COM (short username kudu)
02:11:02.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:02.372670 31817 webserver.cc:492] Webserver started at http://127.30.246.253:43631/ using document root <none> and password file <none>
02:11:02.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:02.373584 31817 fs_manager.cc:362] Metadata directory not provided
02:11:02.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:02.373780 31817 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:11:02.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:02.374176 31817 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
02:11:02.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:02.378779 31817 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-1/data/instance:
02:11:02.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7f8d2c54eaec417cbc734e62b470ab94"
02:11:02.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-24 02:11:02 on dist-test-slave-xv5f"
02:11:02.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:02.379896 31817 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-1/wal/instance:
02:11:02.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7f8d2c54eaec417cbc734e62b470ab94"
02:11:02.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-24 02:11:02 on dist-test-slave-xv5f"
02:11:02.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:02.387634 31817 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.006s sys 0.002s
02:11:02.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:02.393328 31841 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:11:02.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:02.394481 31817 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.001s
02:11:02.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:02.394809 31817 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-1/data,/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-1/wal
02:11:02.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7f8d2c54eaec417cbc734e62b470ab94"
02:11:02.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-24 02:11:02 on dist-test-slave-xv5f"
02:11:02.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:02.395124 31817 fs_report.cc:389] FS layout report
02:11:02.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:11:02.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-1/wal
02:11:02.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-1/wal
02:11:02.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-1/data/data
02:11:02.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:11:02.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:11:02.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:11:02.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:11:02.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:11:02.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:11:02.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:11:02.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:11:02.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:11:02.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:11:02.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:11:02.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:02.449227 31817 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:11:02.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:02.454852 31817 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:11:02.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:02.455277 31817 kserver.cc:163] Server-wide thread pool size limit: 3276
02:11:02.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:02.531847 31817 rpc_server.cc:307] RPC server started. Bound to: 127.30.246.253:36093
02:11:02.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:02.531945 31892 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.30.246.253:36093 every 8 connection(s)
02:11:02.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:02.534677 31817 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-1/data/info.pb
02:11:02.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:02.540711 31707 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskOg0PFu/build/tsan/bin/kudu as pid 31817
02:11:02.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:02.540760 31893 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
02:11:02.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:02.541177 31707 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-1/wal/instance
02:11:02.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.30.246.252@KRBTEST.COM; defaulting to no policy
02:11:02.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:02.556418 31893 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.30.246.254" port: 40109 } has no permanent_uuid. Determining permanent_uuid...
02:11:02.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:11:02.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.30.246.252@KRBTEST.COM" created.
02:11:02.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 24 02:11:02 dist-test-slave-xv5f krb5kdc[31719](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761271862, etypes {rep=17 tkt=17 ses=17}, kudu/127.30.246.253@KRBTEST.COM for kudu/127.30.246.254@KRBTEST.COM
02:11:02.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:11:02.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.30.246.252 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/krb5kdc/kudu.keytab.
02:11:02.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.30.246.252 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/krb5kdc/kudu.keytab.
02:11:02.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:02.595557 31893 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.30.246.253" port: 36093 } has no permanent_uuid. Determining permanent_uuid...
02:11:02.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.30.246.252@KRBTEST.COM; defaulting to no policy
02:11:02.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:11:02.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.30.246.252@KRBTEST.COM" created.
02:11:02.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 24 02:11:02 dist-test-slave-xv5f krb5kdc[31719](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761271862, etypes {rep=17 tkt=17 ses=17}, kudu/127.30.246.253@KRBTEST.COM for kudu/127.30.246.253@KRBTEST.COM
02:11:02.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:02.624305 31893 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.30.246.252" port: 39647 } has no permanent_uuid. Determining permanent_uuid...
02:11:02.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:02.627131 31845 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.30.246.252:39647: connect: Connection refused (error 111)
02:11:02.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:11:02.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.30.246.252 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/krb5kdc/kudu.keytab.
02:11:02.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.30.246.252 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/krb5kdc/kudu.keytab.
02:11:02.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:02.631199 31707 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskOg0PFu/build/tsan/bin/kudu
02:11:02.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskOg0PFu/build/tsan/bin/kudu
02:11:02.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-2/wal
02:11:02.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-2/data
02:11:02.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:11:02.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:11:02.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:11:02.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:11:02.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:11:02.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:11:02.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-2/logs
02:11:02.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-2/data/info.pb
02:11:02.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:11:02.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:11:02.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:11:02.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:11:02.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:11:02.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:11:02.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:11:02.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:11:02.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:11:02.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:11:02.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:11:02.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:11:02.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.246.252:39647
02:11:02.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.246.252
02:11:02.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:11:02.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/krb5kdc/kudu.keytab
02:11:02.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.30.246.252
02:11:02.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:11:02.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
02:11:02.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
02:11:02.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
02:11:02.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.246.212:35299
02:11:02.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:11:02.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:11:02.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:11:02.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:11:02.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.30.246.254:40109,127.30.246.253:36093,127.30.246.252:39647
02:11:02.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:11:02.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42087/jwks.json
02:11:02.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
02:11:02.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
02:11:02.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:02.638111 31893 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.30.246.252:39647: Network error: Client connection negotiation failed: client connection to 127.30.246.252:39647: connect: Connection refused (error 111)
02:11:02.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:02.687785 31893 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.246.252" port: 39647 } attempt: 1
02:11:02.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:02.691851 31893 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.30.246.252:39647: Network error: Client connection negotiation failed: client connection to 127.30.246.252:39647: connect: Connection refused (error 111)
02:11:02.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:02.728770 31808 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.246.253" port: 36093 } attempt: 7
02:11:02.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 24 02:11:02 dist-test-slave-xv5f krb5kdc[31719](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761271860, etypes {rep=17 tkt=17 ses=17}, kudu/127.30.246.254@KRBTEST.COM for kudu/127.30.246.253@KRBTEST.COM
02:11:02.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:02.753386 31808 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.30.246.252" port: 39647 } has no permanent_uuid. Determining permanent_uuid...
02:11:02.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:02.759182 31808 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.30.246.252:39647: Network error: Client connection negotiation failed: client connection to 127.30.246.252:39647: connect: Connection refused (error 111)
02:11:02.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:02.760316 31893 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.246.252" port: 39647 } attempt: 2
02:11:02.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:02.771448 31893 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.30.246.252:39647: Network error: Client connection negotiation failed: client connection to 127.30.246.252:39647: connect: Connection refused (error 111)
02:11:02.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:02.817565 31808 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.246.252" port: 39647 } attempt: 1
02:11:02.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:02.821314 31808 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.30.246.252:39647: Network error: Client connection negotiation failed: client connection to 127.30.246.252:39647: connect: Connection refused (error 111)
02:11:02.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:02.862828 31893 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.246.252" port: 39647 } attempt: 3
02:11:02.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:02.866739 31893 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.30.246.252:39647: Network error: Client connection negotiation failed: client connection to 127.30.246.252:39647: connect: Connection refused (error 111)
02:11:02.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:02.902823 31808 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.246.252" port: 39647 } attempt: 2
02:11:02.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:02.906805 31808 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.30.246.252:39647: Network error: Client connection negotiation failed: client connection to 127.30.246.252:39647: connect: Connection refused (error 111)
02:11:02.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:02.960654 31903 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:11:02.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:02.961220 31903 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:11:02.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:02.961884 31903 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:11:02.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:02.992290 31808 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.246.252" port: 39647 } attempt: 3
02:11:02.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:02.996698 31808 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.30.246.252:39647: Network error: Client connection negotiation failed: client connection to 127.30.246.252:39647: connect: Connection refused (error 111)
02:11:02.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:02.997406 31903 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
02:11:02.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:02.997617 31903 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
02:11:02.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:02.997887 31903 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:11:02.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:02.998100 31903 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
02:11:02.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:02.998298 31903 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
02:11:02.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:02.998557 31903 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
02:11:02.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:02.998726 31903 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:42087/jwks.json
02:11:03.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:03.010226 31893 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.246.252" port: 39647 } attempt: 4
02:11:03.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:03.014386 31893 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.30.246.252:39647: Network error: Client connection negotiation failed: client connection to 127.30.246.252:39647: connect: Connection refused (error 111)
02:11:03.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:03.039718 31903 master_runner.cc:386] Master server non-default flags:
02:11:03.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:11:03.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.246.212:35299
02:11:03.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:11:03.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:11:03.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-2/data
02:11:03.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-2/wal
02:11:03.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
02:11:03.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:11:03.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.30.246.254:40109,127.30.246.253:36093,127.30.246.252:39647
02:11:03.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/krb5kdc/kudu.keytab
02:11:03.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.30.246.252
02:11:03.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:11:03.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:11:03.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:11:03.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.246.252:39647
02:11:03.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:11:03.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:11:03.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:11:03.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42087/jwks.json
02:11:03.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:11:03.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
02:11:03.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
02:11:03.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:11:03.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:11:03.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-2/data/info.pb
02:11:03.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.246.252
02:11:03.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:11:03.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:11:03.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:11:03.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:11:03.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:11:03.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:11:03.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-2/logs
02:11:03.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:11:03.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:11:03.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:11:03.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:11:03.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:11:03.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 5688a9afc53de46b2b2f6847bc5217f38e926fe8
02:11:03.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:11:03.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 24 Oct 2025 01:43:20 UTC on 5fd53c4cbb9d
02:11:03.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8689
02:11:03.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:11:03.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:03.041064 31903 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:11:03.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:03.042114 31903 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:11:03.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:03.053537 31911 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:11:03.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:03.137178 31808 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.246.252" port: 39647 } attempt: 4
02:11:03.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:03.140883 31808 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.30.246.252:39647: Network error: Client connection negotiation failed: client connection to 127.30.246.252:39647: connect: Connection refused (error 111)
02:11:03.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:03.313858 31893 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.246.252" port: 39647 } attempt: 5
02:11:03.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:03.317499 31893 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.30.246.252:39647: Network error: Client connection negotiation failed: client connection to 127.30.246.252:39647: connect: Connection refused (error 111)
02:11:03.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:03.424417 31808 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.246.252" port: 39647 } attempt: 5
02:11:03.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:03.482179 31808 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.30.246.252:39647: Network error: Client connection negotiation failed: client connection to 127.30.246.252:39647: connect: Connection refused (error 111)
02:11:03.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:03.864939 31893 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.246.252" port: 39647 } attempt: 6
02:11:03.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:03.877029 31893 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.30.246.252:39647: Network error: Client connection negotiation failed: client connection to 127.30.246.252:39647: connect: Connection refused (error 111)
02:11:04.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:04.034904 31808 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.246.252" port: 39647 } attempt: 6
02:11:04.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:04.047952 31808 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.30.246.252:39647: Network error: Client connection negotiation failed: client connection to 127.30.246.252:39647: connect: Connection refused (error 111)
02:11:04.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:03.055533 31912 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:11:04.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:04.455857 31913 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1400 milliseconds
02:11:04.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:04.457971 31914 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:11:04.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:04.457975 31903 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:11:04.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:04.461088 31903 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:11:04.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:04.463847 31903 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:11:04.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:04.465250 31903 hybrid_clock.cc:648] HybridClock initialized: now 1761271864465204 us; error 47 us; skew 500 ppm
02:11:04.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:04.466112 31903 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:11:04.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 24 02:11:04 dist-test-slave-xv5f krb5kdc[31719](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761271864, etypes {rep=17 tkt=17 ses=17}, kudu/127.30.246.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:11:04.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:04.472396 31903 init.cc:377] Logged in from keytab as kudu/127.30.246.252@KRBTEST.COM (short username kudu)
02:11:04.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:04.477958 31903 webserver.cc:492] Webserver started at http://127.30.246.252:37959/ using document root <none> and password file <none>
02:11:04.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:04.479002 31903 fs_manager.cc:362] Metadata directory not provided
02:11:04.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:04.479233 31903 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:11:04.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:04.479794 31903 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
02:11:04.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:04.484140 31903 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-2/data/instance:
02:11:04.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6d709ea9b98741149dfde6be4ec3747d"
02:11:04.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-24 02:11:04 on dist-test-slave-xv5f"
02:11:04.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:04.485365 31903 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-2/wal/instance:
02:11:04.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6d709ea9b98741149dfde6be4ec3747d"
02:11:04.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-24 02:11:04 on dist-test-slave-xv5f"
02:11:04.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:04.493407 31903 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.009s sys 0.000s
02:11:04.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:04.499933 31931 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:11:04.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:04.501314 31903 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.005s sys 0.000s
02:11:04.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:04.501670 31903 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-2/data,/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-2/wal
02:11:04.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6d709ea9b98741149dfde6be4ec3747d"
02:11:04.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-24 02:11:04 on dist-test-slave-xv5f"
02:11:04.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:04.502058 31903 fs_report.cc:389] FS layout report
02:11:04.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:11:04.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-2/wal
02:11:04.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-2/wal
02:11:04.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-2/data/data
02:11:04.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:11:04.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:11:04.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:11:04.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:11:04.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:11:04.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:11:04.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:11:04.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:11:04.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:11:04.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:11:04.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:11:04.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:04.505317 31409 debug-util.cc:398] Leaking SignalData structure 0x7b08000b2680 after lost signal to thread 31257
02:11:04.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:04.506223 31409 debug-util.cc:398] Leaking SignalData structure 0x7b08000ad500 after lost signal to thread 31412
02:11:04.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:04.575887 31903 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:11:04.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:04.581936 31903 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:11:04.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:04.582484 31903 kserver.cc:163] Server-wide thread pool size limit: 3276
02:11:04.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:04.679464 31903 rpc_server.cc:307] RPC server started. Bound to: 127.30.246.252:39647
02:11:04.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:04.679543 31982 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.30.246.252:39647 every 8 connection(s)
02:11:04.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:04.682369 31903 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-2/data/info.pb
02:11:04.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:04.684521 31707 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskOg0PFu/build/tsan/bin/kudu as pid 31903
02:11:04.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:04.685052 31707 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-2/wal/instance
02:11:04.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:04.697384 31983 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
02:11:04.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:04.716722 31983 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.30.246.254" port: 40109 } has no permanent_uuid. Determining permanent_uuid...
02:11:04.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 24 02:11:04 dist-test-slave-xv5f krb5kdc[31719](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761271857, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.30.246.252@KRBTEST.COM
02:11:04.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 24 02:11:04 dist-test-slave-xv5f krb5kdc[31719](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761271864, etypes {rep=17 tkt=17 ses=17}, kudu/127.30.246.252@KRBTEST.COM for kudu/127.30.246.254@KRBTEST.COM
02:11:04.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:04.787981 31983 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.30.246.253" port: 36093 } has no permanent_uuid. Determining permanent_uuid...
02:11:04.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 24 02:11:04 dist-test-slave-xv5f krb5kdc[31719](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761271857, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.30.246.253@KRBTEST.COM
02:11:04.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 24 02:11:04 dist-test-slave-xv5f krb5kdc[31719](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761271864, etypes {rep=17 tkt=17 ses=17}, kudu/127.30.246.252@KRBTEST.COM for kudu/127.30.246.253@KRBTEST.COM
02:11:04.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:04.820564 31983 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.30.246.252" port: 39647 } has no permanent_uuid. Determining permanent_uuid...
02:11:04.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 24 02:11:04 dist-test-slave-xv5f krb5kdc[31719](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761271857, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.30.246.254@KRBTEST.COM
02:11:04.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 24 02:11:04 dist-test-slave-xv5f krb5kdc[31719](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761271864, etypes {rep=17 tkt=17 ses=17}, kudu/127.30.246.252@KRBTEST.COM for kudu/127.30.246.252@KRBTEST.COM
02:11:04.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:04.847043 31707 external_mini_cluster.cc:949] 0 TS(s) registered with all masters
02:11:04.855 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:11:04.856 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
02:11:04.857 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
02:11:04.857 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
02:11:04.858 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.30.246.254"
port: 40109
}
bound_http_address {
host: "127.30.246.254"
port: 33217
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.30.246.253"
port: 36093
}
bound_http_address {
host: "127.30.246.253"
port: 43631
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.30.246.252"
port: 39647
}
bound_http_address {
host: "127.30.246.252"
port: 37959
}
}
}
02:11:04.858 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
02:11:04.858 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:11:04.858 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
02:11:04.859 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
02:11:04.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:04.862823 31983 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6d709ea9b98741149dfde6be4ec3747d: Bootstrap starting.
02:11:04.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:04.871551 31983 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 6d709ea9b98741149dfde6be4ec3747d: Neither blocks nor log segments found. Creating new log.
02:11:04.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:04.874172 31983 log.cc:826] T 00000000000000000000000000000000 P 6d709ea9b98741149dfde6be4ec3747d: Log is configured to *not* fsync() on all Append() calls
02:11:04.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:04.881089 31983 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6d709ea9b98741149dfde6be4ec3747d: No bootstrap required, opened a new log
02:11:04.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 24 02:11:04 dist-test-slave-xv5f krb5kdc[31719](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761271864, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:11:04.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
02:11:04.892 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:11:04.893 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/krb5kdc/krb5cc
02:11:04.894 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
02:11:04.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:04.913422 31983 raft_consensus.cc:359] T 00000000000000000000000000000000 P 6d709ea9b98741149dfde6be4ec3747d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3feaddadcc464a6daa4fb7b3dc17612c" member_type: VOTER last_known_addr { host: "127.30.246.254" port: 40109 } } peers { permanent_uuid: "7f8d2c54eaec417cbc734e62b470ab94" member_type: VOTER last_known_addr { host: "127.30.246.253" port: 36093 } } peers { permanent_uuid: "6d709ea9b98741149dfde6be4ec3747d" member_type: VOTER last_known_addr { host: "127.30.246.252" port: 39647 } }
02:11:04.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:04.914430 31983 raft_consensus.cc:385] T 00000000000000000000000000000000 P 6d709ea9b98741149dfde6be4ec3747d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:11:04.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:04.914757 31983 raft_consensus.cc:740] T 00000000000000000000000000000000 P 6d709ea9b98741149dfde6be4ec3747d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6d709ea9b98741149dfde6be4ec3747d, State: Initialized, Role: FOLLOWER
02:11:04.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:04.915776 31983 consensus_queue.cc:260] T 00000000000000000000000000000000 P 6d709ea9b98741149dfde6be4ec3747d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3feaddadcc464a6daa4fb7b3dc17612c" member_type: VOTER last_known_addr { host: "127.30.246.254" port: 40109 } } peers { permanent_uuid: "7f8d2c54eaec417cbc734e62b470ab94" member_type: VOTER last_known_addr { host: "127.30.246.253" port: 36093 } } peers { permanent_uuid: "6d709ea9b98741149dfde6be4ec3747d" member_type: VOTER last_known_addr { host: "127.30.246.252" port: 39647 } }
02:11:04.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:04.918733 31994 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6d709ea9b98741149dfde6be4ec3747d [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3feaddadcc464a6daa4fb7b3dc17612c" member_type: VOTER last_known_addr { host: "127.30.246.254" port: 40109 } } peers { permanent_uuid: "7f8d2c54eaec417cbc734e62b470ab94" member_type: VOTER last_known_addr { host: "127.30.246.253" port: 36093 } } peers { permanent_uuid: "6d709ea9b98741149dfde6be4ec3747d" member_type: VOTER last_known_addr { host: "127.30.246.252" port: 39647 } } }
02:11:04.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:04.919377 31994 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6d709ea9b98741149dfde6be4ec3747d [sys.catalog]: This master's current role is: FOLLOWER
02:11:04.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:04.921133 31983 sys_catalog.cc:565] T 00000000000000000000000000000000 P 6d709ea9b98741149dfde6be4ec3747d [sys.catalog]: configured and running, proceeding with master startup.
02:11:04.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:04.937670 31893 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.246.252" port: 39647 } attempt: 7
02:11:04.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:04.938043 31994 raft_consensus.cc:493] T 00000000000000000000000000000000 P 6d709ea9b98741149dfde6be4ec3747d [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:11:04.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:04.938647 31994 raft_consensus.cc:515] T 00000000000000000000000000000000 P 6d709ea9b98741149dfde6be4ec3747d [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3feaddadcc464a6daa4fb7b3dc17612c" member_type: VOTER last_known_addr { host: "127.30.246.254" port: 40109 } } peers { permanent_uuid: "7f8d2c54eaec417cbc734e62b470ab94" member_type: VOTER last_known_addr { host: "127.30.246.253" port: 36093 } } peers { permanent_uuid: "6d709ea9b98741149dfde6be4ec3747d" member_type: VOTER last_known_addr { host: "127.30.246.252" port: 39647 } }
02:11:04.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:04.943979 31994 leader_election.cc:290] T 00000000000000000000000000000000 P 6d709ea9b98741149dfde6be4ec3747d [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 3feaddadcc464a6daa4fb7b3dc17612c (127.30.246.254:40109), 7f8d2c54eaec417cbc734e62b470ab94 (127.30.246.253:36093)
02:11:04.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:04.969413 32006 catalog_manager.cc:805] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
02:11:04.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 24 02:11:04 dist-test-slave-xv5f krb5kdc[31719](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761271862, etypes {rep=17 tkt=17 ses=17}, kudu/127.30.246.253@KRBTEST.COM for kudu/127.30.246.252@KRBTEST.COM
02:11:05.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:04.974599 31903 master_runner.cc:186] Error getting master registration for 127.30.246.254:40109: OK, instance_id { permanent_uuid: "3feaddadcc464a6daa4fb7b3dc17612c" instance_seqno: 1761271860317274 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
02:11:05.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:05.011284 31903 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
02:11:05.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:05.013973 31893 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 7f8d2c54eaec417cbc734e62b470ab94: Bootstrap starting.
02:11:05.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:05.020205 31893 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 7f8d2c54eaec417cbc734e62b470ab94: Neither blocks nor log segments found. Creating new log.
02:11:05.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:05.022554 31893 log.cc:826] T 00000000000000000000000000000000 P 7f8d2c54eaec417cbc734e62b470ab94: Log is configured to *not* fsync() on all Append() calls
02:11:05.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:05.028138 31893 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 7f8d2c54eaec417cbc734e62b470ab94: No bootstrap required, opened a new log
02:11:05.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:05.048642 31893 raft_consensus.cc:359] T 00000000000000000000000000000000 P 7f8d2c54eaec417cbc734e62b470ab94 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3feaddadcc464a6daa4fb7b3dc17612c" member_type: VOTER last_known_addr { host: "127.30.246.254" port: 40109 } } peers { permanent_uuid: "7f8d2c54eaec417cbc734e62b470ab94" member_type: VOTER last_known_addr { host: "127.30.246.253" port: 36093 } } peers { permanent_uuid: "6d709ea9b98741149dfde6be4ec3747d" member_type: VOTER last_known_addr { host: "127.30.246.252" port: 39647 } }
02:11:05.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:05.049321 31893 raft_consensus.cc:385] T 00000000000000000000000000000000 P 7f8d2c54eaec417cbc734e62b470ab94 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:11:05.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:05.049605 31893 raft_consensus.cc:740] T 00000000000000000000000000000000 P 7f8d2c54eaec417cbc734e62b470ab94 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7f8d2c54eaec417cbc734e62b470ab94, State: Initialized, Role: FOLLOWER
02:11:05.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:05.050582 31893 consensus_queue.cc:260] T 00000000000000000000000000000000 P 7f8d2c54eaec417cbc734e62b470ab94 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3feaddadcc464a6daa4fb7b3dc17612c" member_type: VOTER last_known_addr { host: "127.30.246.254" port: 40109 } } peers { permanent_uuid: "7f8d2c54eaec417cbc734e62b470ab94" member_type: VOTER last_known_addr { host: "127.30.246.253" port: 36093 } } peers { permanent_uuid: "6d709ea9b98741149dfde6be4ec3747d" member_type: VOTER last_known_addr { host: "127.30.246.252" port: 39647 } }
02:11:05.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:05.055012 32008 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7f8d2c54eaec417cbc734e62b470ab94 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3feaddadcc464a6daa4fb7b3dc17612c" member_type: VOTER last_known_addr { host: "127.30.246.254" port: 40109 } } peers { permanent_uuid: "7f8d2c54eaec417cbc734e62b470ab94" member_type: VOTER last_known_addr { host: "127.30.246.253" port: 36093 } } peers { permanent_uuid: "6d709ea9b98741149dfde6be4ec3747d" member_type: VOTER last_known_addr { host: "127.30.246.252" port: 39647 } } }
02:11:05.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:05.056455 31893 sys_catalog.cc:565] T 00000000000000000000000000000000 P 7f8d2c54eaec417cbc734e62b470ab94 [sys.catalog]: configured and running, proceeding with master startup.
02:11:05.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:05.056648 31868 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "6d709ea9b98741149dfde6be4ec3747d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7f8d2c54eaec417cbc734e62b470ab94" is_pre_election: true
02:11:05.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:05.056710 32008 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7f8d2c54eaec417cbc734e62b470ab94 [sys.catalog]: This master's current role is: FOLLOWER
02:11:05.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:05.057619 31868 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 7f8d2c54eaec417cbc734e62b470ab94 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6d709ea9b98741149dfde6be4ec3747d in term 0.
02:11:05.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:05.059314 31934 leader_election.cc:304] T 00000000000000000000000000000000 P 6d709ea9b98741149dfde6be4ec3747d [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6d709ea9b98741149dfde6be4ec3747d, 7f8d2c54eaec417cbc734e62b470ab94; no voters:
02:11:05.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:05.060418 31994 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 6d709ea9b98741149dfde6be4ec3747d [term 0 FOLLOWER]: Leader pre-election won for term 1
02:11:05.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:05.060940 31994 raft_consensus.cc:493] T 00000000000000000000000000000000 P 6d709ea9b98741149dfde6be4ec3747d [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
02:11:05.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:05.061460 31994 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 6d709ea9b98741149dfde6be4ec3747d [term 0 FOLLOWER]: Advancing to term 1
02:11:05.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:05.072043 31994 raft_consensus.cc:515] T 00000000000000000000000000000000 P 6d709ea9b98741149dfde6be4ec3747d [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3feaddadcc464a6daa4fb7b3dc17612c" member_type: VOTER last_known_addr { host: "127.30.246.254" port: 40109 } } peers { permanent_uuid: "7f8d2c54eaec417cbc734e62b470ab94" member_type: VOTER last_known_addr { host: "127.30.246.253" port: 36093 } } peers { permanent_uuid: "6d709ea9b98741149dfde6be4ec3747d" member_type: VOTER last_known_addr { host: "127.30.246.252" port: 39647 } }
02:11:05.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:05.074792 31994 leader_election.cc:290] T 00000000000000000000000000000000 P 6d709ea9b98741149dfde6be4ec3747d [CANDIDATE]: Term 1 election: Requested vote from peers 3feaddadcc464a6daa4fb7b3dc17612c (127.30.246.254:40109), 7f8d2c54eaec417cbc734e62b470ab94 (127.30.246.253:36093)
02:11:05.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:05.077359 31868 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "6d709ea9b98741149dfde6be4ec3747d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7f8d2c54eaec417cbc734e62b470ab94"
02:11:05.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:05.078029 31868 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 7f8d2c54eaec417cbc734e62b470ab94 [term 0 FOLLOWER]: Advancing to term 1
02:11:05.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:05.081625 31808 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.246.252" port: 39647 } attempt: 7
02:11:05.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:05.088799 31937 tablet.cc:2378] T 00000000000000000000000000000000 P 6d709ea9b98741149dfde6be4ec3747d: Can't schedule compaction. Clean time has not been advanced past its initial value.
02:11:05.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:05.110752 31868 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 7f8d2c54eaec417cbc734e62b470ab94 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6d709ea9b98741149dfde6be4ec3747d in term 1.
02:11:05.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:05.115418 31934 leader_election.cc:304] T 00000000000000000000000000000000 P 6d709ea9b98741149dfde6be4ec3747d [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6d709ea9b98741149dfde6be4ec3747d, 7f8d2c54eaec417cbc734e62b470ab94; no voters:
02:11:05.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:05.118093 31994 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 6d709ea9b98741149dfde6be4ec3747d [term 1 FOLLOWER]: Leader election won for term 1
02:11:05.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 24 02:11:05 dist-test-slave-xv5f krb5kdc[31719](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761271860, etypes {rep=17 tkt=17 ses=17}, kudu/127.30.246.254@KRBTEST.COM for kudu/127.30.246.252@KRBTEST.COM
02:11:05.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:05.131124 31994 raft_consensus.cc:697] T 00000000000000000000000000000000 P 6d709ea9b98741149dfde6be4ec3747d [term 1 LEADER]: Becoming Leader. State: Replica: 6d709ea9b98741149dfde6be4ec3747d, State: Running, Role: LEADER
02:11:05.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:05.134204 31994 consensus_queue.cc:237] T 00000000000000000000000000000000 P 6d709ea9b98741149dfde6be4ec3747d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3feaddadcc464a6daa4fb7b3dc17612c" member_type: VOTER last_known_addr { host: "127.30.246.254" port: 40109 } } peers { permanent_uuid: "7f8d2c54eaec417cbc734e62b470ab94" member_type: VOTER last_known_addr { host: "127.30.246.253" port: 36093 } } peers { permanent_uuid: "6d709ea9b98741149dfde6be4ec3747d" member_type: VOTER last_known_addr { host: "127.30.246.252" port: 39647 } }
02:11:05.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:05.169740 31808 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3feaddadcc464a6daa4fb7b3dc17612c: Bootstrap starting.
02:11:05.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:05.182847 32016 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6d709ea9b98741149dfde6be4ec3747d [sys.catalog]: SysCatalogTable state changed. Reason: New leader 6d709ea9b98741149dfde6be4ec3747d. Latest consensus state: current_term: 1 leader_uuid: "6d709ea9b98741149dfde6be4ec3747d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3feaddadcc464a6daa4fb7b3dc17612c" member_type: VOTER last_known_addr { host: "127.30.246.254" port: 40109 } } peers { permanent_uuid: "7f8d2c54eaec417cbc734e62b470ab94" member_type: VOTER last_known_addr { host: "127.30.246.253" port: 36093 } } peers { permanent_uuid: "6d709ea9b98741149dfde6be4ec3747d" member_type: VOTER last_known_addr { host: "127.30.246.252" port: 39647 } } }
02:11:05.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:05.183802 32016 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6d709ea9b98741149dfde6be4ec3747d [sys.catalog]: This master's current role is: LEADER
02:11:05.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:05.185617 31808 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 3feaddadcc464a6daa4fb7b3dc17612c: Neither blocks nor log segments found. Creating new log.
02:11:05.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:05.186096 32022 catalog_manager.cc:805] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
02:11:05.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:05.188700 31808 log.cc:826] T 00000000000000000000000000000000 P 3feaddadcc464a6daa4fb7b3dc17612c: Log is configured to *not* fsync() on all Append() calls
02:11:05.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:05.192754 32023 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
02:11:05.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:05.206861 31808 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3feaddadcc464a6daa4fb7b3dc17612c: No bootstrap required, opened a new log
02:11:05.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:05.213001 31847 tablet.cc:2378] T 00000000000000000000000000000000 P 7f8d2c54eaec417cbc734e62b470ab94: Can't schedule compaction. Clean time has not been advanced past its initial value.
02:11:05.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:05.231648 32023 catalog_manager.cc:1494] Initializing Kudu cluster ID...
02:11:05.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:05.190562 31817 master_runner.cc:186] Error getting master registration for 127.30.246.254:40109: OK, instance_id { permanent_uuid: "3feaddadcc464a6daa4fb7b3dc17612c" instance_seqno: 1761271860317274 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
02:11:05.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:05.240836 31817 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
02:11:05.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:05.271871 31808 raft_consensus.cc:359] T 00000000000000000000000000000000 P 3feaddadcc464a6daa4fb7b3dc17612c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3feaddadcc464a6daa4fb7b3dc17612c" member_type: VOTER last_known_addr { host: "127.30.246.254" port: 40109 } } peers { permanent_uuid: "7f8d2c54eaec417cbc734e62b470ab94" member_type: VOTER last_known_addr { host: "127.30.246.253" port: 36093 } } peers { permanent_uuid: "6d709ea9b98741149dfde6be4ec3747d" member_type: VOTER last_known_addr { host: "127.30.246.252" port: 39647 } }
02:11:05.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:05.273782 31808 raft_consensus.cc:385] T 00000000000000000000000000000000 P 3feaddadcc464a6daa4fb7b3dc17612c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:11:05.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:05.274509 31808 raft_consensus.cc:740] T 00000000000000000000000000000000 P 3feaddadcc464a6daa4fb7b3dc17612c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3feaddadcc464a6daa4fb7b3dc17612c, State: Initialized, Role: FOLLOWER
02:11:05.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:05.276178 31868 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 7f8d2c54eaec417cbc734e62b470ab94 [term 1 FOLLOWER]: Refusing update from remote peer 6d709ea9b98741149dfde6be4ec3747d: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
02:11:05.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:05.276391 31808 consensus_queue.cc:260] T 00000000000000000000000000000000 P 3feaddadcc464a6daa4fb7b3dc17612c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3feaddadcc464a6daa4fb7b3dc17612c" member_type: VOTER last_known_addr { host: "127.30.246.254" port: 40109 } } peers { permanent_uuid: "7f8d2c54eaec417cbc734e62b470ab94" member_type: VOTER last_known_addr { host: "127.30.246.253" port: 36093 } } peers { permanent_uuid: "6d709ea9b98741149dfde6be4ec3747d" member_type: VOTER last_known_addr { host: "127.30.246.252" port: 39647 } }
02:11:05.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:05.278383 31994 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 6d709ea9b98741149dfde6be4ec3747d [LEADER]: Connected to new peer: Peer: permanent_uuid: "7f8d2c54eaec417cbc734e62b470ab94" member_type: VOTER last_known_addr { host: "127.30.246.253" port: 36093 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
02:11:05.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:05.295676 32027 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3feaddadcc464a6daa4fb7b3dc17612c [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3feaddadcc464a6daa4fb7b3dc17612c" member_type: VOTER last_known_addr { host: "127.30.246.254" port: 40109 } } peers { permanent_uuid: "7f8d2c54eaec417cbc734e62b470ab94" member_type: VOTER last_known_addr { host: "127.30.246.253" port: 36093 } } peers { permanent_uuid: "6d709ea9b98741149dfde6be4ec3747d" member_type: VOTER last_known_addr { host: "127.30.246.252" port: 39647 } } }
02:11:05.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:05.296681 32027 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3feaddadcc464a6daa4fb7b3dc17612c [sys.catalog]: This master's current role is: FOLLOWER
02:11:05.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:05.302323 31781 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 3feaddadcc464a6daa4fb7b3dc17612c [term 0 FOLLOWER]: Advancing to term 1
02:11:05.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:05.308226 31782 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "6d709ea9b98741149dfde6be4ec3747d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3feaddadcc464a6daa4fb7b3dc17612c"
02:11:05.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:05.313290 31808 sys_catalog.cc:565] T 00000000000000000000000000000000 P 3feaddadcc464a6daa4fb7b3dc17612c [sys.catalog]: configured and running, proceeding with master startup.
02:11:05.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:05.314782 32008 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7f8d2c54eaec417cbc734e62b470ab94 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 6d709ea9b98741149dfde6be4ec3747d. Latest consensus state: current_term: 1 leader_uuid: "6d709ea9b98741149dfde6be4ec3747d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3feaddadcc464a6daa4fb7b3dc17612c" member_type: VOTER last_known_addr { host: "127.30.246.254" port: 40109 } } peers { permanent_uuid: "7f8d2c54eaec417cbc734e62b470ab94" member_type: VOTER last_known_addr { host: "127.30.246.253" port: 36093 } } peers { permanent_uuid: "6d709ea9b98741149dfde6be4ec3747d" member_type: VOTER last_known_addr { host: "127.30.246.252" port: 39647 } } }
02:11:05.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:05.315358 31783 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "6d709ea9b98741149dfde6be4ec3747d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3feaddadcc464a6daa4fb7b3dc17612c" is_pre_election: true
02:11:05.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:05.315804 32008 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7f8d2c54eaec417cbc734e62b470ab94 [sys.catalog]: This master's current role is: FOLLOWER
02:11:05.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:05.320935 31781 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 3feaddadcc464a6daa4fb7b3dc17612c [term 1 FOLLOWER]: Refusing update from remote peer 6d709ea9b98741149dfde6be4ec3747d: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
02:11:05.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:05.323096 31994 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 6d709ea9b98741149dfde6be4ec3747d [LEADER]: Connected to new peer: Peer: permanent_uuid: "3feaddadcc464a6daa4fb7b3dc17612c" member_type: VOTER last_known_addr { host: "127.30.246.254" port: 40109 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
02:11:05.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:05.331933 32028 mvcc.cc:204] Tried to move back new op lower bound from 7214169560122421248 to 7214169559666728960. Current Snapshot: MvccSnapshot[applied={T|T < 7214169560122421248}]
02:11:05.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:05.336668 32008 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7f8d2c54eaec417cbc734e62b470ab94 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "6d709ea9b98741149dfde6be4ec3747d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3feaddadcc464a6daa4fb7b3dc17612c" member_type: VOTER last_known_addr { host: "127.30.246.254" port: 40109 } } peers { permanent_uuid: "7f8d2c54eaec417cbc734e62b470ab94" member_type: VOTER last_known_addr { host: "127.30.246.253" port: 36093 } } peers { permanent_uuid: "6d709ea9b98741149dfde6be4ec3747d" member_type: VOTER last_known_addr { host: "127.30.246.252" port: 39647 } } }
02:11:05.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:05.337414 32008 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7f8d2c54eaec417cbc734e62b470ab94 [sys.catalog]: This master's current role is: FOLLOWER
02:11:05.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:05.348059 31994 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6d709ea9b98741149dfde6be4ec3747d [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "6d709ea9b98741149dfde6be4ec3747d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3feaddadcc464a6daa4fb7b3dc17612c" member_type: VOTER last_known_addr { host: "127.30.246.254" port: 40109 } } peers { permanent_uuid: "7f8d2c54eaec417cbc734e62b470ab94" member_type: VOTER last_known_addr { host: "127.30.246.253" port: 36093 } } peers { permanent_uuid: "6d709ea9b98741149dfde6be4ec3747d" member_type: VOTER last_known_addr { host: "127.30.246.252" port: 39647 } } }
02:11:05.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:05.348865 31994 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6d709ea9b98741149dfde6be4ec3747d [sys.catalog]: This master's current role is: LEADER
02:11:05.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:05.350139 32023 catalog_manager.cc:1357] Generated new cluster ID: 8be6e7d7c50a40e1b40d161833e26bd8
02:11:05.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:05.350626 32023 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
02:11:05.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:05.360638 32027 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3feaddadcc464a6daa4fb7b3dc17612c [sys.catalog]: SysCatalogTable state changed. Reason: New leader 6d709ea9b98741149dfde6be4ec3747d. Latest consensus state: current_term: 1 leader_uuid: "6d709ea9b98741149dfde6be4ec3747d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3feaddadcc464a6daa4fb7b3dc17612c" member_type: VOTER last_known_addr { host: "127.30.246.254" port: 40109 } } peers { permanent_uuid: "7f8d2c54eaec417cbc734e62b470ab94" member_type: VOTER last_known_addr { host: "127.30.246.253" port: 36093 } } peers { permanent_uuid: "6d709ea9b98741149dfde6be4ec3747d" member_type: VOTER last_known_addr { host: "127.30.246.252" port: 39647 } } }
02:11:05.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:05.360610 32016 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6d709ea9b98741149dfde6be4ec3747d [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "6d709ea9b98741149dfde6be4ec3747d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3feaddadcc464a6daa4fb7b3dc17612c" member_type: VOTER last_known_addr { host: "127.30.246.254" port: 40109 } } peers { permanent_uuid: "7f8d2c54eaec417cbc734e62b470ab94" member_type: VOTER last_known_addr { host: "127.30.246.253" port: 36093 } } peers { permanent_uuid: "6d709ea9b98741149dfde6be4ec3747d" member_type: VOTER last_known_addr { host: "127.30.246.252" port: 39647 } } }
02:11:05.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:05.361407 32016 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6d709ea9b98741149dfde6be4ec3747d [sys.catalog]: This master's current role is: LEADER
02:11:05.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:05.361397 32027 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3feaddadcc464a6daa4fb7b3dc17612c [sys.catalog]: This master's current role is: FOLLOWER
02:11:05.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:05.394026 32027 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3feaddadcc464a6daa4fb7b3dc17612c [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "6d709ea9b98741149dfde6be4ec3747d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3feaddadcc464a6daa4fb7b3dc17612c" member_type: VOTER last_known_addr { host: "127.30.246.254" port: 40109 } } peers { permanent_uuid: "7f8d2c54eaec417cbc734e62b470ab94" member_type: VOTER last_known_addr { host: "127.30.246.253" port: 36093 } } peers { permanent_uuid: "6d709ea9b98741149dfde6be4ec3747d" member_type: VOTER last_known_addr { host: "127.30.246.252" port: 39647 } } }
02:11:05.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:05.394842 32027 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3feaddadcc464a6daa4fb7b3dc17612c [sys.catalog]: This master's current role is: FOLLOWER
02:11:05.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:05.420315 32023 catalog_manager.cc:1380] Generated new certificate authority record
02:11:05.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:05.422667 32023 catalog_manager.cc:1514] Loading token signing keys...
02:11:05.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:05.490463 32045 catalog_manager.cc:1269] Loaded cluster ID: 8be6e7d7c50a40e1b40d161833e26bd8
02:11:05.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:05.490749 32045 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 3feaddadcc464a6daa4fb7b3dc17612c: loading cluster ID for follower catalog manager: success
02:11:05.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:05.509557 32023 catalog_manager.cc:6022] T 00000000000000000000000000000000 P 6d709ea9b98741149dfde6be4ec3747d: Generated new TSK 0
02:11:05.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:05.510710 32023 catalog_manager.cc:1524] Initializing in-progress tserver states...
02:11:05.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:05.512454 32045 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 3feaddadcc464a6daa4fb7b3dc17612c: acquiring CA information for follower catalog manager: success
02:11:05.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:05.532269 32045 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 3feaddadcc464a6daa4fb7b3dc17612c: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
02:11:05.895 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.30.246.253:36093
02:11:05.896 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
02:11:05.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:05.897640 31707 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskOg0PFu/build/tsan/bin/kudu with pid 31817
02:11:05.947 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:11:05.947 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.30.246.254:40109
02:11:05.947 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
02:11:05.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:05.948195 31707 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskOg0PFu/build/tsan/bin/kudu with pid 31736
02:11:05.988 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:11:05.988 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.30.246.252:39647
02:11:05.988 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
02:11:05.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:05.988646 31707 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskOg0PFu/build/tsan/bin/kudu with pid 31903
02:11:06.018 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:11:06.018 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.30.246.253:36093
02:11:06.018 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
02:11:06.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:06.019846 31707 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskOg0PFu/build/tsan/bin/kudu
02:11:06.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskOg0PFu/build/tsan/bin/kudu
02:11:06.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-1/wal
02:11:06.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-1/data
02:11:06.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:11:06.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:11:06.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:11:06.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:11:06.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:11:06.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:11:06.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-1/logs
02:11:06.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-1/data/info.pb
02:11:06.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:11:06.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:11:06.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:11:06.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:11:06.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:11:06.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:11:06.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:11:06.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:11:06.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:11:06.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:11:06.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:11:06.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:11:06.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.246.253:36093
02:11:06.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.246.253
02:11:06.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=43631
02:11:06.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/krb5kdc/kudu.keytab
02:11:06.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.30.246.253
02:11:06.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:11:06.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
02:11:06.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
02:11:06.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
02:11:06.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.246.212:35299
02:11:06.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:11:06.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:11:06.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:11:06.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:11:06.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.30.246.254:40109,127.30.246.253:36093,127.30.246.252:39647
02:11:06.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:11:06.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42087/jwks.json
02:11:06.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
02:11:06.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
02:11:06.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:06.370816 32049 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:11:06.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:06.371474 32049 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:11:06.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:06.372202 32049 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:11:06.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:06.407197 32049 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
02:11:06.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:06.407491 32049 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
02:11:06.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:06.407882 32049 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:11:06.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:06.408177 32049 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
02:11:06.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:06.408449 32049 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
02:11:06.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:06.408746 32049 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
02:11:06.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:06.409054 32049 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:42087/jwks.json
02:11:06.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:06.449805 32049 master_runner.cc:386] Master server non-default flags:
02:11:06.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:11:06.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.246.212:35299
02:11:06.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:11:06.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:11:06.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-1/data
02:11:06.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-1/wal
02:11:06.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
02:11:06.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:11:06.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.30.246.254:40109,127.30.246.253:36093,127.30.246.252:39647
02:11:06.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/krb5kdc/kudu.keytab
02:11:06.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.30.246.253
02:11:06.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:11:06.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:11:06.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:11:06.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.246.253:36093
02:11:06.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:11:06.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:11:06.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:11:06.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42087/jwks.json
02:11:06.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:11:06.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
02:11:06.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
02:11:06.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:11:06.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:11:06.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-1/data/info.pb
02:11:06.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.246.253
02:11:06.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=43631
02:11:06.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:11:06.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:11:06.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:11:06.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:11:06.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:11:06.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-1/logs
02:11:06.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:11:06.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:11:06.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:11:06.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:11:06.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:11:06.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 5688a9afc53de46b2b2f6847bc5217f38e926fe8
02:11:06.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:11:06.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 24 Oct 2025 01:43:20 UTC on 5fd53c4cbb9d
02:11:06.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8689
02:11:06.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:11:06.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:06.451646 32049 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:11:06.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:06.453042 32049 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:11:06.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:06.466578 32055 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:11:08.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:07.869256 32054 debug-util.cc:398] Leaking SignalData structure 0x7b0800027080 after lost signal to thread 32049
02:11:08.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:08.338503 32054 kernel_stack_watchdog.cc:198] Thread 32049 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 398ms:
02:11:08.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
02:11:08.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
02:11:08.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
02:11:08.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
02:11:08.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:06.467703 32056 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:11:08.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:08.339645 32049 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.872s user 0.690s sys 1.112s
02:11:08.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:08.339697 32057 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1870 milliseconds
02:11:08.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:08.340113 32049 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.872s user 0.690s sys 1.112s
02:11:08.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:08.341229 32049 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:11:08.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:08.341261 32058 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:11:08.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:08.345650 32049 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:11:08.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:08.348209 32049 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:11:08.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:08.349653 32049 hybrid_clock.cc:648] HybridClock initialized: now 1761271868349603 us; error 55 us; skew 500 ppm
02:11:08.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:08.350627 32049 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:11:08.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 24 02:11:08 dist-test-slave-xv5f krb5kdc[31719](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761271868, etypes {rep=17 tkt=17 ses=17}, kudu/127.30.246.253@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:11:08.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:08.357235 32049 init.cc:377] Logged in from keytab as kudu/127.30.246.253@KRBTEST.COM (short username kudu)
02:11:08.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:08.362138 32049 webserver.cc:492] Webserver started at http://127.30.246.253:43631/ using document root <none> and password file <none>
02:11:08.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:08.363188 32049 fs_manager.cc:362] Metadata directory not provided
02:11:08.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:08.363432 32049 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:11:08.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:08.371820 32049 fs_manager.cc:714] Time spent opening directory manager: real 0.005s user 0.004s sys 0.000s
02:11:08.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:08.376791 32067 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:11:08.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:08.378063 32049 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.003s
02:11:08.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:08.378386 32049 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-1/data,/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-1/wal
02:11:08.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7f8d2c54eaec417cbc734e62b470ab94"
02:11:08.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-24 02:11:02 on dist-test-slave-xv5f"
02:11:08.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:08.380478 32049 fs_report.cc:389] FS layout report
02:11:08.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:11:08.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-1/wal
02:11:08.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-1/wal
02:11:08.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-1/data/data
02:11:08.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:11:08.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:11:08.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:11:08.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:11:08.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:11:08.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:11:08.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:11:08.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:11:08.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:11:08.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:11:08.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:11:08.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:08.435123 32049 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:11:08.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:08.440963 32049 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:11:08.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:08.441406 32049 kserver.cc:163] Server-wide thread pool size limit: 3276
02:11:08.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:08.516894 32049 rpc_server.cc:307] RPC server started. Bound to: 127.30.246.253:36093
02:11:08.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:08.516950 32118 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.30.246.253:36093 every 8 connection(s)
02:11:08.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:08.519847 32049 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-1/data/info.pb
02:11:08.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:08.524283 31707 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskOg0PFu/build/tsan/bin/kudu as pid 32049
02:11:08.524 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:11:08.525 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.30.246.254:40109
02:11:08.525 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
02:11:08.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:08.526862 31707 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskOg0PFu/build/tsan/bin/kudu
02:11:08.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskOg0PFu/build/tsan/bin/kudu
02:11:08.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-0/wal
02:11:08.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-0/data
02:11:08.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:11:08.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:11:08.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:11:08.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:11:08.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:11:08.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:11:08.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-0/logs
02:11:08.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-0/data/info.pb
02:11:08.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:11:08.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:11:08.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:11:08.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:11:08.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:11:08.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:11:08.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:11:08.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:11:08.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:11:08.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:11:08.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:11:08.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:11:08.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.246.254:40109
02:11:08.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.246.254
02:11:08.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33217
02:11:08.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/krb5kdc/kudu.keytab
02:11:08.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.30.246.254
02:11:08.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:11:08.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
02:11:08.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
02:11:08.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
02:11:08.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.246.212:35299
02:11:08.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:11:08.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:11:08.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:11:08.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:11:08.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.30.246.254:40109,127.30.246.253:36093,127.30.246.252:39647
02:11:08.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:11:08.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42087/jwks.json
02:11:08.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
02:11:08.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
02:11:08.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:08.535140 32119 sys_catalog.cc:263] Verifying existing consensus state
02:11:08.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:08.540899 32119 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 7f8d2c54eaec417cbc734e62b470ab94: Bootstrap starting.
02:11:08.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:08.581251 32119 log.cc:826] T 00000000000000000000000000000000 P 7f8d2c54eaec417cbc734e62b470ab94: Log is configured to *not* fsync() on all Append() calls
02:11:08.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:08.598379 32119 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 7f8d2c54eaec417cbc734e62b470ab94: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
02:11:08.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:08.599215 32119 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 7f8d2c54eaec417cbc734e62b470ab94: Bootstrap complete.
02:11:08.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:08.625193 32119 raft_consensus.cc:359] T 00000000000000000000000000000000 P 7f8d2c54eaec417cbc734e62b470ab94 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3feaddadcc464a6daa4fb7b3dc17612c" member_type: VOTER last_known_addr { host: "127.30.246.254" port: 40109 } } peers { permanent_uuid: "7f8d2c54eaec417cbc734e62b470ab94" member_type: VOTER last_known_addr { host: "127.30.246.253" port: 36093 } } peers { permanent_uuid: "6d709ea9b98741149dfde6be4ec3747d" member_type: VOTER last_known_addr { host: "127.30.246.252" port: 39647 } }
02:11:08.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:08.627640 32119 raft_consensus.cc:740] T 00000000000000000000000000000000 P 7f8d2c54eaec417cbc734e62b470ab94 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7f8d2c54eaec417cbc734e62b470ab94, State: Initialized, Role: FOLLOWER
02:11:08.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:08.628403 32119 consensus_queue.cc:260] T 00000000000000000000000000000000 P 7f8d2c54eaec417cbc734e62b470ab94 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3feaddadcc464a6daa4fb7b3dc17612c" member_type: VOTER last_known_addr { host: "127.30.246.254" port: 40109 } } peers { permanent_uuid: "7f8d2c54eaec417cbc734e62b470ab94" member_type: VOTER last_known_addr { host: "127.30.246.253" port: 36093 } } peers { permanent_uuid: "6d709ea9b98741149dfde6be4ec3747d" member_type: VOTER last_known_addr { host: "127.30.246.252" port: 39647 } }
02:11:08.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:08.631196 32123 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7f8d2c54eaec417cbc734e62b470ab94 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3feaddadcc464a6daa4fb7b3dc17612c" member_type: VOTER last_known_addr { host: "127.30.246.254" port: 40109 } } peers { permanent_uuid: "7f8d2c54eaec417cbc734e62b470ab94" member_type: VOTER last_known_addr { host: "127.30.246.253" port: 36093 } } peers { permanent_uuid: "6d709ea9b98741149dfde6be4ec3747d" member_type: VOTER last_known_addr { host: "127.30.246.252" port: 39647 } } }
02:11:08.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:08.632052 32123 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7f8d2c54eaec417cbc734e62b470ab94 [sys.catalog]: This master's current role is: FOLLOWER
02:11:08.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:08.633145 32119 sys_catalog.cc:565] T 00000000000000000000000000000000 P 7f8d2c54eaec417cbc734e62b470ab94 [sys.catalog]: configured and running, proceeding with master startup.
02:11:08.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:08.670478 32134 catalog_manager.cc:1269] Loaded cluster ID: 8be6e7d7c50a40e1b40d161833e26bd8
02:11:08.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:08.670768 32134 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 7f8d2c54eaec417cbc734e62b470ab94: loading cluster ID for follower catalog manager: success
02:11:08.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:08.678845 32134 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 7f8d2c54eaec417cbc734e62b470ab94: acquiring CA information for follower catalog manager: success
02:11:08.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:08.684664 32134 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 7f8d2c54eaec417cbc734e62b470ab94: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
02:11:08.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:08.660621 32049 master_runner.cc:186] Error getting master registration for 127.30.246.254:40109: Network error: Client connection negotiation failed: client connection to 127.30.246.254:40109: connect: Connection refused (error 111),
02:11:08.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:08.688133 32049 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
02:11:08.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:08.904647 32121 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:11:08.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:08.905236 32121 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:11:08.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:08.905886 32121 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:11:08.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:08.948156 32121 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
02:11:08.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:08.948500 32121 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
02:11:08.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:08.948957 32121 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:11:08.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:08.949292 32121 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
02:11:08.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:08.949623 32121 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
02:11:08.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:08.949959 32121 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
02:11:08.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:08.950238 32121 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:42087/jwks.json
02:11:08.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:08.998749 32121 master_runner.cc:386] Master server non-default flags:
02:11:08.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:11:08.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.246.212:35299
02:11:08.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:11:08.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:11:08.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-0/data
02:11:08.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-0/wal
02:11:08.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
02:11:08.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:11:09.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.30.246.254:40109,127.30.246.253:36093,127.30.246.252:39647
02:11:09.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/krb5kdc/kudu.keytab
02:11:09.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.30.246.254
02:11:09.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:11:09.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:11:09.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:11:09.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.246.254:40109
02:11:09.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:11:09.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:11:09.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:11:09.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42087/jwks.json
02:11:09.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:11:09.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
02:11:09.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
02:11:09.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:11:09.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:11:09.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-0/data/info.pb
02:11:09.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.246.254
02:11:09.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33217
02:11:09.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:11:09.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:11:09.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:11:09.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:11:09.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:11:09.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-0/logs
02:11:09.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:11:09.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:11:09.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:11:09.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:11:09.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:11:09.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 5688a9afc53de46b2b2f6847bc5217f38e926fe8
02:11:09.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:11:09.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 24 Oct 2025 01:43:20 UTC on 5fd53c4cbb9d
02:11:09.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8689
02:11:09.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:11:09.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:08.999993 32121 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:11:09.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:09.001220 32121 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:11:09.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:09.014050 32143 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:11:10.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:10.228339 32152 raft_consensus.cc:493] T 00000000000000000000000000000000 P 7f8d2c54eaec417cbc734e62b470ab94 [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:11:10.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:10.229192 32152 raft_consensus.cc:515] T 00000000000000000000000000000000 P 7f8d2c54eaec417cbc734e62b470ab94 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3feaddadcc464a6daa4fb7b3dc17612c" member_type: VOTER last_known_addr { host: "127.30.246.254" port: 40109 } } peers { permanent_uuid: "7f8d2c54eaec417cbc734e62b470ab94" member_type: VOTER last_known_addr { host: "127.30.246.253" port: 36093 } } peers { permanent_uuid: "6d709ea9b98741149dfde6be4ec3747d" member_type: VOTER last_known_addr { host: "127.30.246.252" port: 39647 } }
02:11:10.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:10.263374 32152 leader_election.cc:290] T 00000000000000000000000000000000 P 7f8d2c54eaec417cbc734e62b470ab94 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 3feaddadcc464a6daa4fb7b3dc17612c (127.30.246.254:40109), 6d709ea9b98741149dfde6be4ec3747d (127.30.246.252:39647)
02:11:10.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:10.275884 32071 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.30.246.252:39647: connect: Connection refused (error 111)
02:11:10.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:10.295977 32070 leader_election.cc:336] T 00000000000000000000000000000000 P 7f8d2c54eaec417cbc734e62b470ab94 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 3feaddadcc464a6daa4fb7b3dc17612c (127.30.246.254:40109): Network error: Client connection negotiation failed: client connection to 127.30.246.254:40109: connect: Connection refused (error 111)
02:11:10.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:10.296746 32071 leader_election.cc:336] T 00000000000000000000000000000000 P 7f8d2c54eaec417cbc734e62b470ab94 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 6d709ea9b98741149dfde6be4ec3747d (127.30.246.252:39647): Network error: Client connection negotiation failed: client connection to 127.30.246.252:39647: connect: Connection refused (error 111)
02:11:10.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:10.303625 32071 leader_election.cc:304] T 00000000000000000000000000000000 P 7f8d2c54eaec417cbc734e62b470ab94 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 7f8d2c54eaec417cbc734e62b470ab94; no voters: 3feaddadcc464a6daa4fb7b3dc17612c, 6d709ea9b98741149dfde6be4ec3747d
02:11:10.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:10.310892 32152 raft_consensus.cc:2749] T 00000000000000000000000000000000 P 7f8d2c54eaec417cbc734e62b470ab94 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
02:11:11.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:10.417973 32142 debug-util.cc:398] Leaking SignalData structure 0x7b0800027080 after lost signal to thread 32121
02:11:11.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:11.195770 32142 kernel_stack_watchdog.cc:198] Thread 32121 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 399ms:
02:11:11.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
02:11:11.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
02:11:11.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
02:11:11.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
02:11:11.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:11.196869 32121 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 2.181s user 0.611s sys 1.133s
02:11:11.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:11.197289 32121 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 2.182s user 0.611s sys 1.133s
02:11:11.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:09.015529 32144 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:11:11.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:11.199476 32146 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:11:11.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:11.204633 32145 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2187 milliseconds
02:11:11.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:11.204661 32121 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:11:11.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:11.206130 32121 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:11:11.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:11.208657 32121 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:11:11.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:11.210059 32121 hybrid_clock.cc:648] HybridClock initialized: now 1761271871209997 us; error 103 us; skew 500 ppm
02:11:11.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:11.211122 32121 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:11:11.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 24 02:11:11 dist-test-slave-xv5f krb5kdc[31719](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761271871, etypes {rep=17 tkt=17 ses=17}, kudu/127.30.246.254@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:11:11.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:11.220973 32121 init.cc:377] Logged in from keytab as kudu/127.30.246.254@KRBTEST.COM (short username kudu)
02:11:11.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:11.242098 32121 webserver.cc:492] Webserver started at http://127.30.246.254:33217/ using document root <none> and password file <none>
02:11:11.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:11.244382 32121 fs_manager.cc:362] Metadata directory not provided
02:11:11.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:11.244725 32121 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:11:11.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:11.277320 32121 fs_manager.cc:714] Time spent opening directory manager: real 0.028s user 0.022s sys 0.004s
02:11:11.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:11.286777 32164 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:11:11.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:11.288580 32121 fs_manager.cc:730] Time spent opening block manager: real 0.008s user 0.007s sys 0.001s
02:11:11.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:11.289007 32121 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-0/data,/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-0/wal
02:11:11.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3feaddadcc464a6daa4fb7b3dc17612c"
02:11:11.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-24 02:11:00 on dist-test-slave-xv5f"
02:11:11.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:11.292004 32121 fs_report.cc:389] FS layout report
02:11:11.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:11:11.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-0/wal
02:11:11.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-0/wal
02:11:11.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-0/data/data
02:11:11.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:11:11.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:11:11.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:11:11.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:11:11.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:11:11.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:11:11.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:11:11.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:11:11.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:11:11.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:11:11.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:11:11.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:11.359294 32121 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:11:11.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:11.369336 32121 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:11:11.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:11.369874 32121 kserver.cc:163] Server-wide thread pool size limit: 3276
02:11:11.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:11.473863 32215 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.30.246.254:40109 every 8 connection(s)
02:11:11.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:11.474601 32121 rpc_server.cc:307] RPC server started. Bound to: 127.30.246.254:40109
02:11:11.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:11.479727 32121 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-0/data/info.pb
02:11:11.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:11.480248 31707 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskOg0PFu/build/tsan/bin/kudu as pid 32121
02:11:11.481 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:11:11.481 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.30.246.252:39647
02:11:11.481 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
02:11:11.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:11.482995 31707 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskOg0PFu/build/tsan/bin/kudu
02:11:11.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskOg0PFu/build/tsan/bin/kudu
02:11:11.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-2/wal
02:11:11.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-2/data
02:11:11.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:11:11.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:11:11.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:11:11.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:11:11.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:11:11.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:11:11.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-2/logs
02:11:11.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-2/data/info.pb
02:11:11.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:11:11.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:11:11.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:11:11.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:11:11.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:11:11.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:11:11.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:11:11.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:11:11.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:11:11.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:11:11.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:11:11.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:11:11.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.246.252:39647
02:11:11.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.246.252
02:11:11.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37959
02:11:11.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/krb5kdc/kudu.keytab
02:11:11.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.30.246.252
02:11:11.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:11:11.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
02:11:11.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
02:11:11.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
02:11:11.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.246.212:35299
02:11:11.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:11:11.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:11:11.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:11:11.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:11:11.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.30.246.254:40109,127.30.246.253:36093,127.30.246.252:39647
02:11:11.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:11:11.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42087/jwks.json
02:11:11.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
02:11:11.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
02:11:11.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:11.501114 32216 sys_catalog.cc:263] Verifying existing consensus state
02:11:11.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:11.507694 32216 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3feaddadcc464a6daa4fb7b3dc17612c: Bootstrap starting.
02:11:11.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:11.558486 32216 log.cc:826] T 00000000000000000000000000000000 P 3feaddadcc464a6daa4fb7b3dc17612c: Log is configured to *not* fsync() on all Append() calls
02:11:11.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:11.575958 32216 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3feaddadcc464a6daa4fb7b3dc17612c: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
02:11:11.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:11.576838 32216 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3feaddadcc464a6daa4fb7b3dc17612c: Bootstrap complete.
02:11:11.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:11.602608 32216 raft_consensus.cc:359] T 00000000000000000000000000000000 P 3feaddadcc464a6daa4fb7b3dc17612c [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3feaddadcc464a6daa4fb7b3dc17612c" member_type: VOTER last_known_addr { host: "127.30.246.254" port: 40109 } } peers { permanent_uuid: "7f8d2c54eaec417cbc734e62b470ab94" member_type: VOTER last_known_addr { host: "127.30.246.253" port: 36093 } } peers { permanent_uuid: "6d709ea9b98741149dfde6be4ec3747d" member_type: VOTER last_known_addr { host: "127.30.246.252" port: 39647 } }
02:11:11.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:11.604943 32216 raft_consensus.cc:740] T 00000000000000000000000000000000 P 3feaddadcc464a6daa4fb7b3dc17612c [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3feaddadcc464a6daa4fb7b3dc17612c, State: Initialized, Role: FOLLOWER
02:11:11.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:11.606129 32216 consensus_queue.cc:260] T 00000000000000000000000000000000 P 3feaddadcc464a6daa4fb7b3dc17612c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3feaddadcc464a6daa4fb7b3dc17612c" member_type: VOTER last_known_addr { host: "127.30.246.254" port: 40109 } } peers { permanent_uuid: "7f8d2c54eaec417cbc734e62b470ab94" member_type: VOTER last_known_addr { host: "127.30.246.253" port: 36093 } } peers { permanent_uuid: "6d709ea9b98741149dfde6be4ec3747d" member_type: VOTER last_known_addr { host: "127.30.246.252" port: 39647 } }
02:11:11.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:11.609220 32220 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3feaddadcc464a6daa4fb7b3dc17612c [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3feaddadcc464a6daa4fb7b3dc17612c" member_type: VOTER last_known_addr { host: "127.30.246.254" port: 40109 } } peers { permanent_uuid: "7f8d2c54eaec417cbc734e62b470ab94" member_type: VOTER last_known_addr { host: "127.30.246.253" port: 36093 } } peers { permanent_uuid: "6d709ea9b98741149dfde6be4ec3747d" member_type: VOTER last_known_addr { host: "127.30.246.252" port: 39647 } } }
02:11:11.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:11.610476 32220 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3feaddadcc464a6daa4fb7b3dc17612c [sys.catalog]: This master's current role is: FOLLOWER
02:11:11.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:11.611336 32216 sys_catalog.cc:565] T 00000000000000000000000000000000 P 3feaddadcc464a6daa4fb7b3dc17612c [sys.catalog]: configured and running, proceeding with master startup.
02:11:11.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:11.656193 32231 catalog_manager.cc:1269] Loaded cluster ID: 8be6e7d7c50a40e1b40d161833e26bd8
02:11:11.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:11.656906 32231 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 3feaddadcc464a6daa4fb7b3dc17612c: loading cluster ID for follower catalog manager: success
02:11:11.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:11.664984 32231 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 3feaddadcc464a6daa4fb7b3dc17612c: acquiring CA information for follower catalog manager: success
02:11:11.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 24 02:11:11 dist-test-slave-xv5f krb5kdc[31719](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761271871, etypes {rep=17 tkt=17 ses=17}, kudu/127.30.246.254@KRBTEST.COM for kudu/127.30.246.254@KRBTEST.COM
02:11:11.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:11.672421 32231 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 3feaddadcc464a6daa4fb7b3dc17612c: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
02:11:11.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:11.757545 32121 master_runner.cc:186] Error getting master registration for 127.30.246.252:39647: Network error: Client connection negotiation failed: client connection to 127.30.246.252:39647: connect: Connection refused (error 111),
02:11:11.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:11.832001 32121 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
02:11:11.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:11.990156 32218 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:11:11.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:11.990962 32218 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:11:11.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:11.991796 32218 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:11:12.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:12.037621 32218 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
02:11:12.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:12.037961 32218 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
02:11:12.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:12.038470 32218 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:11:12.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:12.038817 32218 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
02:11:12.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:12.039160 32218 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
02:11:12.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:12.039489 32218 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
02:11:12.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:12.039741 32218 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:42087/jwks.json
02:11:12.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:12.108125 32218 master_runner.cc:386] Master server non-default flags:
02:11:12.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:11:12.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.246.212:35299
02:11:12.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:11:12.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:11:12.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-2/data
02:11:12.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-2/wal
02:11:12.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
02:11:12.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:11:12.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.30.246.254:40109,127.30.246.253:36093,127.30.246.252:39647
02:11:12.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/krb5kdc/kudu.keytab
02:11:12.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.30.246.252
02:11:12.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:11:12.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:11:12.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:11:12.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.246.252:39647
02:11:12.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:11:12.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:11:12.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:11:12.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42087/jwks.json
02:11:12.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:11:12.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
02:11:12.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
02:11:12.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:11:12.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:11:12.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-2/data/info.pb
02:11:12.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.246.252
02:11:12.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37959
02:11:12.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:11:12.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:11:12.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:11:12.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:11:12.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:11:12.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-2/logs
02:11:12.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:11:12.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:11:12.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:11:12.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:11:12.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:11:12.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 5688a9afc53de46b2b2f6847bc5217f38e926fe8
02:11:12.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:11:12.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 24 Oct 2025 01:43:20 UTC on 5fd53c4cbb9d
02:11:12.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8689
02:11:12.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:11:12.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:12.109808 32218 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:11:12.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:12.111256 32218 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:11:12.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:12.128595 32243 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:11:12.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:12.311621 32248 raft_consensus.cc:493] T 00000000000000000000000000000000 P 7f8d2c54eaec417cbc734e62b470ab94 [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:11:12.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:12.312815 32248 raft_consensus.cc:515] T 00000000000000000000000000000000 P 7f8d2c54eaec417cbc734e62b470ab94 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3feaddadcc464a6daa4fb7b3dc17612c" member_type: VOTER last_known_addr { host: "127.30.246.254" port: 40109 } } peers { permanent_uuid: "7f8d2c54eaec417cbc734e62b470ab94" member_type: VOTER last_known_addr { host: "127.30.246.253" port: 36093 } } peers { permanent_uuid: "6d709ea9b98741149dfde6be4ec3747d" member_type: VOTER last_known_addr { host: "127.30.246.252" port: 39647 } }
02:11:12.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:12.356788 32248 leader_election.cc:290] T 00000000000000000000000000000000 P 7f8d2c54eaec417cbc734e62b470ab94 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 3feaddadcc464a6daa4fb7b3dc17612c (127.30.246.254:40109), 6d709ea9b98741149dfde6be4ec3747d (127.30.246.252:39647)
02:11:12.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:12.393849 32071 leader_election.cc:336] T 00000000000000000000000000000000 P 7f8d2c54eaec417cbc734e62b470ab94 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 6d709ea9b98741149dfde6be4ec3747d (127.30.246.252:39647): Network error: Client connection negotiation failed: client connection to 127.30.246.252:39647: connect: Connection refused (error 111)
02:11:12.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:12.422760 32191 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "7f8d2c54eaec417cbc734e62b470ab94" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "3feaddadcc464a6daa4fb7b3dc17612c" is_pre_election: true
02:11:12.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:12.424165 32191 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 3feaddadcc464a6daa4fb7b3dc17612c [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7f8d2c54eaec417cbc734e62b470ab94 in term 1.
02:11:12.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:12.427048 32070 leader_election.cc:304] T 00000000000000000000000000000000 P 7f8d2c54eaec417cbc734e62b470ab94 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 3feaddadcc464a6daa4fb7b3dc17612c, 7f8d2c54eaec417cbc734e62b470ab94; no voters: 6d709ea9b98741149dfde6be4ec3747d
02:11:12.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:12.428603 32248 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 7f8d2c54eaec417cbc734e62b470ab94 [term 1 FOLLOWER]: Leader pre-election won for term 2
02:11:12.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:12.429112 32248 raft_consensus.cc:493] T 00000000000000000000000000000000 P 7f8d2c54eaec417cbc734e62b470ab94 [term 1 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
02:11:12.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:12.429551 32248 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 7f8d2c54eaec417cbc734e62b470ab94 [term 1 FOLLOWER]: Advancing to term 2
02:11:12.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:12.464793 32248 raft_consensus.cc:515] T 00000000000000000000000000000000 P 7f8d2c54eaec417cbc734e62b470ab94 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3feaddadcc464a6daa4fb7b3dc17612c" member_type: VOTER last_known_addr { host: "127.30.246.254" port: 40109 } } peers { permanent_uuid: "7f8d2c54eaec417cbc734e62b470ab94" member_type: VOTER last_known_addr { host: "127.30.246.253" port: 36093 } } peers { permanent_uuid: "6d709ea9b98741149dfde6be4ec3747d" member_type: VOTER last_known_addr { host: "127.30.246.252" port: 39647 } }
02:11:12.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:12.484483 32191 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "7f8d2c54eaec417cbc734e62b470ab94" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "3feaddadcc464a6daa4fb7b3dc17612c"
02:11:12.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:12.485294 32191 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 3feaddadcc464a6daa4fb7b3dc17612c [term 1 FOLLOWER]: Advancing to term 2
02:11:12.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:12.502664 32071 leader_election.cc:336] T 00000000000000000000000000000000 P 7f8d2c54eaec417cbc734e62b470ab94 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 6d709ea9b98741149dfde6be4ec3747d (127.30.246.252:39647): Network error: Client connection negotiation failed: client connection to 127.30.246.252:39647: connect: Connection refused (error 111)
02:11:12.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:12.503916 32248 leader_election.cc:290] T 00000000000000000000000000000000 P 7f8d2c54eaec417cbc734e62b470ab94 [CANDIDATE]: Term 2 election: Requested vote from peers 3feaddadcc464a6daa4fb7b3dc17612c (127.30.246.254:40109), 6d709ea9b98741149dfde6be4ec3747d (127.30.246.252:39647)
02:11:12.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:12.517079 32191 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 3feaddadcc464a6daa4fb7b3dc17612c [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7f8d2c54eaec417cbc734e62b470ab94 in term 2.
02:11:12.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:12.523069 32070 leader_election.cc:304] T 00000000000000000000000000000000 P 7f8d2c54eaec417cbc734e62b470ab94 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 3feaddadcc464a6daa4fb7b3dc17612c, 7f8d2c54eaec417cbc734e62b470ab94; no voters: 6d709ea9b98741149dfde6be4ec3747d
02:11:12.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:12.524657 32248 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 7f8d2c54eaec417cbc734e62b470ab94 [term 2 FOLLOWER]: Leader election won for term 2
02:11:12.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:12.543865 32248 raft_consensus.cc:697] T 00000000000000000000000000000000 P 7f8d2c54eaec417cbc734e62b470ab94 [term 2 LEADER]: Becoming Leader. State: Replica: 7f8d2c54eaec417cbc734e62b470ab94, State: Running, Role: LEADER
02:11:12.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:12.545472 32248 consensus_queue.cc:237] T 00000000000000000000000000000000 P 7f8d2c54eaec417cbc734e62b470ab94 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3feaddadcc464a6daa4fb7b3dc17612c" member_type: VOTER last_known_addr { host: "127.30.246.254" port: 40109 } } peers { permanent_uuid: "7f8d2c54eaec417cbc734e62b470ab94" member_type: VOTER last_known_addr { host: "127.30.246.253" port: 36093 } } peers { permanent_uuid: "6d709ea9b98741149dfde6be4ec3747d" member_type: VOTER last_known_addr { host: "127.30.246.252" port: 39647 } }
02:11:12.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:12.571635 32253 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7f8d2c54eaec417cbc734e62b470ab94 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 7f8d2c54eaec417cbc734e62b470ab94. Latest consensus state: current_term: 2 leader_uuid: "7f8d2c54eaec417cbc734e62b470ab94" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3feaddadcc464a6daa4fb7b3dc17612c" member_type: VOTER last_known_addr { host: "127.30.246.254" port: 40109 } } peers { permanent_uuid: "7f8d2c54eaec417cbc734e62b470ab94" member_type: VOTER last_known_addr { host: "127.30.246.253" port: 36093 } } peers { permanent_uuid: "6d709ea9b98741149dfde6be4ec3747d" member_type: VOTER last_known_addr { host: "127.30.246.252" port: 39647 } } }
02:11:12.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:12.573689 32253 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7f8d2c54eaec417cbc734e62b470ab94 [sys.catalog]: This master's current role is: LEADER
02:11:12.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:12.577605 32255 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
02:11:12.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:12.597349 32255 catalog_manager.cc:1494] Initializing Kudu cluster ID...
02:11:12.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:12.604470 32255 catalog_manager.cc:1269] Loaded cluster ID: 8be6e7d7c50a40e1b40d161833e26bd8
02:11:12.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:12.604785 32255 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
02:11:12.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:12.609434 32255 catalog_manager.cc:1514] Loading token signing keys...
02:11:12.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:12.622476 32255 catalog_manager.cc:6033] T 00000000000000000000000000000000 P 7f8d2c54eaec417cbc734e62b470ab94: Loaded TSK: 0
02:11:12.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:12.623831 32255 catalog_manager.cc:1524] Initializing in-progress tserver states...
02:11:13.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:13.177969 32191 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 3feaddadcc464a6daa4fb7b3dc17612c [term 2 FOLLOWER]: Refusing update from remote peer 7f8d2c54eaec417cbc734e62b470ab94: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 2 index: 5. (index mismatch)
02:11:13.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:13.181353 32256 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 7f8d2c54eaec417cbc734e62b470ab94 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3feaddadcc464a6daa4fb7b3dc17612c" member_type: VOTER last_known_addr { host: "127.30.246.254" port: 40109 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.001s
02:11:13.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:13.211269 32071 consensus_peers.cc:597] T 00000000000000000000000000000000 P 7f8d2c54eaec417cbc734e62b470ab94 -> Peer 6d709ea9b98741149dfde6be4ec3747d (127.30.246.252:39647): Couldn't send request to peer 6d709ea9b98741149dfde6be4ec3747d. Status: Network error: Client connection negotiation failed: client connection to 127.30.246.252:39647: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
02:11:13.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:13.220155 32264 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3feaddadcc464a6daa4fb7b3dc17612c [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "7f8d2c54eaec417cbc734e62b470ab94" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3feaddadcc464a6daa4fb7b3dc17612c" member_type: VOTER last_known_addr { host: "127.30.246.254" port: 40109 } } peers { permanent_uuid: "7f8d2c54eaec417cbc734e62b470ab94" member_type: VOTER last_known_addr { host: "127.30.246.253" port: 36093 } } peers { permanent_uuid: "6d709ea9b98741149dfde6be4ec3747d" member_type: VOTER last_known_addr { host: "127.30.246.252" port: 39647 } } }
02:11:13.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:13.220968 32264 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3feaddadcc464a6daa4fb7b3dc17612c [sys.catalog]: This master's current role is: FOLLOWER
02:11:13.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:13.223989 32260 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3feaddadcc464a6daa4fb7b3dc17612c [sys.catalog]: SysCatalogTable state changed. Reason: New leader 7f8d2c54eaec417cbc734e62b470ab94. Latest consensus state: current_term: 2 leader_uuid: "7f8d2c54eaec417cbc734e62b470ab94" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3feaddadcc464a6daa4fb7b3dc17612c" member_type: VOTER last_known_addr { host: "127.30.246.254" port: 40109 } } peers { permanent_uuid: "7f8d2c54eaec417cbc734e62b470ab94" member_type: VOTER last_known_addr { host: "127.30.246.253" port: 36093 } } peers { permanent_uuid: "6d709ea9b98741149dfde6be4ec3747d" member_type: VOTER last_known_addr { host: "127.30.246.252" port: 39647 } } }
02:11:13.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:13.225235 32260 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3feaddadcc464a6daa4fb7b3dc17612c [sys.catalog]: This master's current role is: FOLLOWER
02:11:13.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:13.243642 32256 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7f8d2c54eaec417cbc734e62b470ab94 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "7f8d2c54eaec417cbc734e62b470ab94" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3feaddadcc464a6daa4fb7b3dc17612c" member_type: VOTER last_known_addr { host: "127.30.246.254" port: 40109 } } peers { permanent_uuid: "7f8d2c54eaec417cbc734e62b470ab94" member_type: VOTER last_known_addr { host: "127.30.246.253" port: 36093 } } peers { permanent_uuid: "6d709ea9b98741149dfde6be4ec3747d" member_type: VOTER last_known_addr { host: "127.30.246.252" port: 39647 } } }
02:11:13.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:13.244683 32256 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7f8d2c54eaec417cbc734e62b470ab94 [sys.catalog]: This master's current role is: LEADER
02:11:13.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:13.737548 32049 master_runner.cc:186] Error getting master registration for 127.30.246.252:39647: Network error: Client connection negotiation failed: client connection to 127.30.246.252:39647: connect: Connection refused (error 111),
02:11:13.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:13.738758 32049 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
02:11:15.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:13.530166 32241 debug-util.cc:398] Leaking SignalData structure 0x7b0800027080 after lost signal to thread 32218
02:11:15.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:15.140486 32241 kernel_stack_watchdog.cc:198] Thread 32218 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 397ms:
02:11:15.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
02:11:15.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
02:11:15.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
02:11:15.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
02:11:15.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:12.128669 32242 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:11:15.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:15.143330 32218 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 3.013s user 1.023s sys 1.597s
02:11:15.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:15.143695 32218 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 3.014s user 1.023s sys 1.597s
02:11:15.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:15.148198 32244 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 3012 milliseconds
02:11:15.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:15.149492 32245 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:11:15.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:15.149636 32218 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:11:15.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:15.151378 32218 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:11:15.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:15.154547 32218 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:11:15.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:15.156054 32218 hybrid_clock.cc:648] HybridClock initialized: now 1761271875155996 us; error 43 us; skew 500 ppm
02:11:15.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:15.157213 32218 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:11:15.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 24 02:11:15 dist-test-slave-xv5f krb5kdc[31719](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761271875, etypes {rep=17 tkt=17 ses=17}, kudu/127.30.246.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:11:15.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:15.165380 32218 init.cc:377] Logged in from keytab as kudu/127.30.246.252@KRBTEST.COM (short username kudu)
02:11:15.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:15.171438 32218 webserver.cc:492] Webserver started at http://127.30.246.252:37959/ using document root <none> and password file <none>
02:11:15.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:15.172787 32218 fs_manager.cc:362] Metadata directory not provided
02:11:15.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:15.173134 32218 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:11:15.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:15.185431 32218 fs_manager.cc:714] Time spent opening directory manager: real 0.008s user 0.005s sys 0.004s
02:11:15.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:15.191809 32277 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:11:15.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:15.193346 32218 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.002s sys 0.003s
02:11:15.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:15.193809 32218 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-2/data,/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-2/wal
02:11:15.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6d709ea9b98741149dfde6be4ec3747d"
02:11:15.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-24 02:11:04 on dist-test-slave-xv5f"
02:11:15.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:15.196777 32218 fs_report.cc:389] FS layout report
02:11:15.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:11:15.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-2/wal
02:11:15.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-2/wal
02:11:15.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-2/data/data
02:11:15.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:11:15.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:11:15.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:11:15.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:11:15.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:11:15.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:11:15.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:11:15.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:11:15.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:11:15.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:11:15.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:11:15.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:15.272042 32218 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:11:15.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:15.280324 32218 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:11:15.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:15.280936 32218 kserver.cc:163] Server-wide thread pool size limit: 3276
02:11:15.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:15.388940 32218 rpc_server.cc:307] RPC server started. Bound to: 127.30.246.252:39647
02:11:15.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:15.389003 32329 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.30.246.252:39647 every 8 connection(s)
02:11:15.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:15.392231 32218 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-2/data/info.pb
02:11:15.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:15.399494 31707 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskOg0PFu/build/tsan/bin/kudu as pid 32218
02:11:15.400 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:11:15.401 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.30.246.253:36093(127.30.246.253:36093)] connecting to peer
02:11:15.403 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.30.246.254:40109(127.30.246.254:40109)] connecting to peer
02:11:15.404 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.30.246.253:36093(127.30.246.253:36093)] Successfully connected to peer
02:11:15.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:15.406621 32330 sys_catalog.cc:263] Verifying existing consensus state
02:11:15.411 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.30.246.252:39647(127.30.246.252:39647)] connecting to peer
02:11:15.413 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.30.246.252:39647(127.30.246.252:39647)] Successfully connected to peer
02:11:15.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:15.426460 32330 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6d709ea9b98741149dfde6be4ec3747d: Bootstrap starting.
02:11:15.429 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.30.246.254:40109(127.30.246.254:40109)] Successfully connected to peer
02:11:15.431 [DEBUG - kudu-nio-0] (SecurityContext.java:222) Refreshing Kerberos credentials...
02:11:15.431 [DEBUG - kudu-nio-0] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/krb5kdc/krb5cc
02:11:15.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 24 02:11:15 dist-test-slave-xv5f krb5kdc[31719](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761271864, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:11:15.448 [DEBUG - kudu-nio-0] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
02:11:15.452 [INFO - kudu-nio-0] (SecurityContext.java:256) Successfully refreshed Kerberos credentials from ticket cache
02:11:15.454 [DEBUG - kudu-nio-1] (SecurityContext.java:222) Refreshing Kerberos credentials...
02:11:15.454 [DEBUG - kudu-nio-1] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/krb5kdc/krb5cc
02:11:15.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 24 02:11:15 dist-test-slave-xv5f krb5kdc[31719](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761271864, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:11:15.468 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.30.246.253
02:11:15.471 [DEBUG - kudu-nio-1] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
02:11:15.489 [INFO - kudu-nio-1] (SecurityContext.java:256) Successfully refreshed Kerberos credentials from ticket cache
02:11:15.490 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.30.246.254
02:11:15.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:15.495131 32330 log.cc:826] T 00000000000000000000000000000000 P 6d709ea9b98741149dfde6be4ec3747d: Log is configured to *not* fsync() on all Append() calls
02:11:15.506 [DEBUG - kudu-nio-0] (SecurityContext.java:222) Refreshing Kerberos credentials...
02:11:15.506 [DEBUG - kudu-nio-0] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/krb5kdc/krb5cc
02:11:15.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 24 02:11:15 dist-test-slave-xv5f krb5kdc[31719](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761271864, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:11:15.516 [DEBUG - kudu-nio-0] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
02:11:15.522 [INFO - kudu-nio-0] (SecurityContext.java:256) Successfully refreshed Kerberos credentials from ticket cache
02:11:15.525 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.30.246.252
02:11:15.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 24 02:11:15 dist-test-slave-xv5f krb5kdc[31719](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761271864, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.30.246.254@KRBTEST.COM
02:11:15.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:15.579547 32330 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6d709ea9b98741149dfde6be4ec3747d: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
02:11:15.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:15.580669 32330 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6d709ea9b98741149dfde6be4ec3747d: Bootstrap complete.
02:11:15.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 24 02:11:15 dist-test-slave-xv5f krb5kdc[31719](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761271868, etypes {rep=17 tkt=17 ses=17}, kudu/127.30.246.253@KRBTEST.COM for kudu/127.30.246.252@KRBTEST.COM
02:11:15.602 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x50ad09d4, L:/127.0.0.1:34358 - R:/127.30.246.254:40109] using SASL/GSSAPI
02:11:15.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 24 02:11:15 dist-test-slave-xv5f krb5kdc[31719](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761271864, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.30.246.253@KRBTEST.COM
02:11:15.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:15.613958 32330 raft_consensus.cc:359] T 00000000000000000000000000000000 P 6d709ea9b98741149dfde6be4ec3747d [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3feaddadcc464a6daa4fb7b3dc17612c" member_type: VOTER last_known_addr { host: "127.30.246.254" port: 40109 } } peers { permanent_uuid: "7f8d2c54eaec417cbc734e62b470ab94" member_type: VOTER last_known_addr { host: "127.30.246.253" port: 36093 } } peers { permanent_uuid: "6d709ea9b98741149dfde6be4ec3747d" member_type: VOTER last_known_addr { host: "127.30.246.252" port: 39647 } }
02:11:15.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:15.617624 32330 raft_consensus.cc:740] T 00000000000000000000000000000000 P 6d709ea9b98741149dfde6be4ec3747d [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6d709ea9b98741149dfde6be4ec3747d, State: Initialized, Role: FOLLOWER
02:11:15.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:15.620559 32330 consensus_queue.cc:260] T 00000000000000000000000000000000 P 6d709ea9b98741149dfde6be4ec3747d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3feaddadcc464a6daa4fb7b3dc17612c" member_type: VOTER last_known_addr { host: "127.30.246.254" port: 40109 } } peers { permanent_uuid: "7f8d2c54eaec417cbc734e62b470ab94" member_type: VOTER last_known_addr { host: "127.30.246.253" port: 36093 } } peers { permanent_uuid: "6d709ea9b98741149dfde6be4ec3747d" member_type: VOTER last_known_addr { host: "127.30.246.252" port: 39647 } }
02:11:15.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:15.635573 32341 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6d709ea9b98741149dfde6be4ec3747d [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3feaddadcc464a6daa4fb7b3dc17612c" member_type: VOTER last_known_addr { host: "127.30.246.254" port: 40109 } } peers { permanent_uuid: "7f8d2c54eaec417cbc734e62b470ab94" member_type: VOTER last_known_addr { host: "127.30.246.253" port: 36093 } } peers { permanent_uuid: "6d709ea9b98741149dfde6be4ec3747d" member_type: VOTER last_known_addr { host: "127.30.246.252" port: 39647 } } }
02:11:15.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:15.636673 32341 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6d709ea9b98741149dfde6be4ec3747d [sys.catalog]: This master's current role is: FOLLOWER
02:11:15.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:15.637389 32305 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 6d709ea9b98741149dfde6be4ec3747d [term 1 FOLLOWER]: Advancing to term 2
02:11:15.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:15.646042 32330 sys_catalog.cc:565] T 00000000000000000000000000000000 P 6d709ea9b98741149dfde6be4ec3747d [sys.catalog]: configured and running, proceeding with master startup.
02:11:15.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 24 02:11:15 dist-test-slave-xv5f krb5kdc[31719](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761271864, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.30.246.252@KRBTEST.COM
02:11:15.656 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x501cb05f, L:/127.0.0.1:52724 - R:/127.30.246.253:36093] using SASL/GSSAPI
02:11:15.668 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
02:11:15.669 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=ListTabletServers, tablet=null, attempt=1, TimeoutTracker(timeout=30000, elapsed=268), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.246.253:36093, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.246.254:40109, [10ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.246.252:39647, [209ms] Sub RPC ConnectToMaster: received response from server master-127.30.246.254:40109: OK, [266ms] Sub RPC ConnectToMaster: received response from server master-127.30.246.253:36093: OK, deferred=null) after lookup
02:11:15.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:15.676653 32341 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6d709ea9b98741149dfde6be4ec3747d [sys.catalog]: SysCatalogTable state changed. Reason: New leader 7f8d2c54eaec417cbc734e62b470ab94. Latest consensus state: current_term: 2 leader_uuid: "7f8d2c54eaec417cbc734e62b470ab94" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3feaddadcc464a6daa4fb7b3dc17612c" member_type: VOTER last_known_addr { host: "127.30.246.254" port: 40109 } } peers { permanent_uuid: "7f8d2c54eaec417cbc734e62b470ab94" member_type: VOTER last_known_addr { host: "127.30.246.253" port: 36093 } } peers { permanent_uuid: "6d709ea9b98741149dfde6be4ec3747d" member_type: VOTER last_known_addr { host: "127.30.246.252" port: 39647 } } }
02:11:15.679 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x864f570b, L:/127.0.0.1:46942 - R:/127.30.246.252:39647] using SASL/GSSAPI
02:11:15.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:15.678840 32341 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6d709ea9b98741149dfde6be4ec3747d [sys.catalog]: This master's current role is: FOLLOWER
02:11:15.685 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.30.246.252:39647: Catalog manager is not initialized. State: Starting
02:11:15.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:15.687086 32346 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6d709ea9b98741149dfde6be4ec3747d [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "7f8d2c54eaec417cbc734e62b470ab94" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3feaddadcc464a6daa4fb7b3dc17612c" member_type: VOTER last_known_addr { host: "127.30.246.254" port: 40109 } } peers { permanent_uuid: "7f8d2c54eaec417cbc734e62b470ab94" member_type: VOTER last_known_addr { host: "127.30.246.253" port: 36093 } } peers { permanent_uuid: "6d709ea9b98741149dfde6be4ec3747d" member_type: VOTER last_known_addr { host: "127.30.246.252" port: 39647 } } }
02:11:15.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:15.688208 32346 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6d709ea9b98741149dfde6be4ec3747d [sys.catalog]: This master's current role is: FOLLOWER
02:11:15.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:15.719820 32355 catalog_manager.cc:1269] Loaded cluster ID: 8be6e7d7c50a40e1b40d161833e26bd8
02:11:15.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:15.720134 32355 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 6d709ea9b98741149dfde6be4ec3747d: loading cluster ID for follower catalog manager: success
02:11:15.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:15.727265 32355 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 6d709ea9b98741149dfde6be4ec3747d: acquiring CA information for follower catalog manager: success
02:11:15.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:15.733335 32355 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 6d709ea9b98741149dfde6be4ec3747d: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
02:11:15.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 24 02:11:15 dist-test-slave-xv5f krb5kdc[31719](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761271875, etypes {rep=17 tkt=17 ses=17}, kudu/127.30.246.252@KRBTEST.COM for kudu/127.30.246.254@KRBTEST.COM
02:11:15.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:15.986264 32256 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7f8d2c54eaec417cbc734e62b470ab94 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "7f8d2c54eaec417cbc734e62b470ab94" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3feaddadcc464a6daa4fb7b3dc17612c" member_type: VOTER last_known_addr { host: "127.30.246.254" port: 40109 } } peers { permanent_uuid: "7f8d2c54eaec417cbc734e62b470ab94" member_type: VOTER last_known_addr { host: "127.30.246.253" port: 36093 } } peers { permanent_uuid: "6d709ea9b98741149dfde6be4ec3747d" member_type: VOTER last_known_addr { host: "127.30.246.252" port: 39647 } } }
02:11:15.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:15.987600 32256 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7f8d2c54eaec417cbc734e62b470ab94 [sys.catalog]: This master's current role is: LEADER
02:11:16.674 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.30.246.253:36093
02:11:16.675 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
02:11:16.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:16.676054 31707 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskOg0PFu/build/tsan/bin/kudu with pid 32049
02:11:16.699 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.30.246.253:36093(127.30.246.253:36093)] handling channelInactive
02:11:16.699 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.30.246.253:36093(127.30.246.253:36093)] cleaning up while in state READY due to: connection closed
02:11:16.701 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:11:16.701 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.30.246.254:40109
02:11:16.701 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
02:11:16.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:16.701560 31707 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskOg0PFu/build/tsan/bin/kudu with pid 32121
02:11:16.739 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.30.246.254:40109(127.30.246.254:40109)] handling channelInactive
02:11:16.740 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.30.246.254:40109(127.30.246.254:40109)] cleaning up while in state READY due to: connection closed
02:11:16.743 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:11:16.743 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.30.246.252:39647
02:11:16.743 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
02:11:16.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:16.743763 31707 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskOg0PFu/build/tsan/bin/kudu with pid 32218
02:11:16.773 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.30.246.252:39647(127.30.246.252:39647)] handling channelInactive
02:11:16.773 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.30.246.252:39647(127.30.246.252:39647)] cleaning up while in state READY due to: connection closed
02:11:16.773 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:11:16.773 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.30.246.253:36093
02:11:16.774 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
02:11:16.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:16.776021 31707 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskOg0PFu/build/tsan/bin/kudu
02:11:16.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskOg0PFu/build/tsan/bin/kudu
02:11:16.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-1/wal
02:11:16.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-1/data
02:11:16.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:11:16.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:11:16.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:11:16.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:11:16.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:11:16.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:11:16.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-1/logs
02:11:16.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-1/data/info.pb
02:11:16.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:11:16.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:11:16.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:11:16.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:11:16.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:11:16.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:11:16.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:11:16.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:11:16.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:11:16.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:11:16.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:11:16.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:11:16.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.246.253:36093
02:11:16.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.246.253
02:11:16.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=43631
02:11:16.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/krb5kdc/kudu.keytab
02:11:16.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.30.246.253
02:11:16.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:11:16.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
02:11:16.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
02:11:16.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
02:11:16.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.246.212:35299
02:11:16.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:11:16.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:11:16.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:11:16.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:11:16.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.30.246.254:40109,127.30.246.253:36093,127.30.246.252:39647
02:11:16.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:11:16.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42087/jwks.json
02:11:16.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
02:11:16.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
02:11:17.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:17.099707 32363 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:11:17.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:17.100371 32363 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:11:17.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:17.101140 32363 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:11:17.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:17.135656 32363 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
02:11:17.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:17.135946 32363 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
02:11:17.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:17.136359 32363 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:11:17.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:17.136644 32363 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
02:11:17.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:17.136915 32363 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
02:11:17.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:17.137192 32363 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
02:11:17.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:17.137408 32363 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:42087/jwks.json
02:11:17.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:17.177048 32363 master_runner.cc:386] Master server non-default flags:
02:11:17.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:11:17.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.246.212:35299
02:11:17.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:11:17.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:11:17.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-1/data
02:11:17.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-1/wal
02:11:17.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
02:11:17.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:11:17.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.30.246.254:40109,127.30.246.253:36093,127.30.246.252:39647
02:11:17.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/krb5kdc/kudu.keytab
02:11:17.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.30.246.253
02:11:17.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:11:17.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:11:17.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:11:17.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.246.253:36093
02:11:17.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:11:17.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:11:17.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:11:17.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42087/jwks.json
02:11:17.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:11:17.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
02:11:17.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
02:11:17.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:11:17.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:11:17.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-1/data/info.pb
02:11:17.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.246.253
02:11:17.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=43631
02:11:17.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:11:17.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:11:17.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:11:17.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:11:17.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:11:17.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-1/logs
02:11:17.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:11:17.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:11:17.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:11:17.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:11:17.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:11:17.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 5688a9afc53de46b2b2f6847bc5217f38e926fe8
02:11:17.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:11:17.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 24 Oct 2025 01:43:20 UTC on 5fd53c4cbb9d
02:11:17.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8689
02:11:17.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:11:17.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:17.178699 32363 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:11:17.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:17.180198 32363 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:11:17.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:17.192812 32370 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:11:19.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:18.596410 32369 debug-util.cc:398] Leaking SignalData structure 0x7b0800027080 after lost signal to thread 32363
02:11:19.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:19.002995 32363 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.809s user 0.682s sys 1.087s
02:11:19.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:19.003458 32363 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.810s user 0.682s sys 1.087s
02:11:19.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:17.193452 32371 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:11:19.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:19.005535 32373 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:11:19.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:19.006146 32372 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1810 milliseconds
02:11:19.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:19.006260 32363 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:11:19.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:19.010843 32363 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:11:19.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:19.013579 32363 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:11:19.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:19.014991 32363 hybrid_clock.cc:648] HybridClock initialized: now 1761271879014957 us; error 63 us; skew 500 ppm
02:11:19.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:19.015874 32363 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:11:19.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 24 02:11:19 dist-test-slave-xv5f krb5kdc[31719](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761271879, etypes {rep=17 tkt=17 ses=17}, kudu/127.30.246.253@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:11:19.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:19.022336 32363 init.cc:377] Logged in from keytab as kudu/127.30.246.253@KRBTEST.COM (short username kudu)
02:11:19.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:19.027462 32363 webserver.cc:492] Webserver started at http://127.30.246.253:43631/ using document root <none> and password file <none>
02:11:19.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:19.028501 32363 fs_manager.cc:362] Metadata directory not provided
02:11:19.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:19.028748 32363 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:11:19.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:19.036943 32363 fs_manager.cc:714] Time spent opening directory manager: real 0.005s user 0.004s sys 0.002s
02:11:19.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:19.042054 32385 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:11:19.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:19.043357 32363 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.001s sys 0.003s
02:11:19.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:19.043747 32363 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-1/data,/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-1/wal
02:11:19.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7f8d2c54eaec417cbc734e62b470ab94"
02:11:19.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-24 02:11:02 on dist-test-slave-xv5f"
02:11:19.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:19.045789 32363 fs_report.cc:389] FS layout report
02:11:19.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:11:19.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-1/wal
02:11:19.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-1/wal
02:11:19.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-1/data/data
02:11:19.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:11:19.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:11:19.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:11:19.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:11:19.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:11:19.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:11:19.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:11:19.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:11:19.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:11:19.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:11:19.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:11:19.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:19.102864 32363 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:11:19.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:19.112439 32363 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:11:19.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:19.112941 32363 kserver.cc:163] Server-wide thread pool size limit: 3276
02:11:19.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:19.192314 32363 rpc_server.cc:307] RPC server started. Bound to: 127.30.246.253:36093
02:11:19.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:19.192412 32437 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.30.246.253:36093 every 8 connection(s)
02:11:19.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:19.195318 32363 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-1/data/info.pb
02:11:19.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:19.205185 31707 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskOg0PFu/build/tsan/bin/kudu as pid 32363
02:11:19.205 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:11:19.206 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.30.246.254:40109
02:11:19.206 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
02:11:19.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:19.207497 31707 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskOg0PFu/build/tsan/bin/kudu
02:11:19.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskOg0PFu/build/tsan/bin/kudu
02:11:19.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-0/wal
02:11:19.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-0/data
02:11:19.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:11:19.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:11:19.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:11:19.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:11:19.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:11:19.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:11:19.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-0/logs
02:11:19.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-0/data/info.pb
02:11:19.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:11:19.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:11:19.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:11:19.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:11:19.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:11:19.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:11:19.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:11:19.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:11:19.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:11:19.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:11:19.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:11:19.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:11:19.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.246.254:40109
02:11:19.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.246.254
02:11:19.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33217
02:11:19.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/krb5kdc/kudu.keytab
02:11:19.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.30.246.254
02:11:19.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:11:19.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
02:11:19.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
02:11:19.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
02:11:19.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.246.212:35299
02:11:19.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:11:19.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:11:19.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:11:19.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:11:19.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.30.246.254:40109,127.30.246.253:36093,127.30.246.252:39647
02:11:19.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:11:19.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42087/jwks.json
02:11:19.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
02:11:19.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
02:11:19.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:19.208284 32438 sys_catalog.cc:263] Verifying existing consensus state
02:11:19.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:19.214262 32438 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 7f8d2c54eaec417cbc734e62b470ab94: Bootstrap starting.
02:11:19.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:19.254630 32438 log.cc:826] T 00000000000000000000000000000000 P 7f8d2c54eaec417cbc734e62b470ab94: Log is configured to *not* fsync() on all Append() calls
02:11:19.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:19.274044 32438 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 7f8d2c54eaec417cbc734e62b470ab94: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
02:11:19.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:19.274958 32438 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 7f8d2c54eaec417cbc734e62b470ab94: Bootstrap complete.
02:11:19.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:19.300352 32438 raft_consensus.cc:359] T 00000000000000000000000000000000 P 7f8d2c54eaec417cbc734e62b470ab94 [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3feaddadcc464a6daa4fb7b3dc17612c" member_type: VOTER last_known_addr { host: "127.30.246.254" port: 40109 } } peers { permanent_uuid: "7f8d2c54eaec417cbc734e62b470ab94" member_type: VOTER last_known_addr { host: "127.30.246.253" port: 36093 } } peers { permanent_uuid: "6d709ea9b98741149dfde6be4ec3747d" member_type: VOTER last_known_addr { host: "127.30.246.252" port: 39647 } }
02:11:19.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:19.302987 32438 raft_consensus.cc:740] T 00000000000000000000000000000000 P 7f8d2c54eaec417cbc734e62b470ab94 [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7f8d2c54eaec417cbc734e62b470ab94, State: Initialized, Role: FOLLOWER
02:11:19.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:19.304306 32438 consensus_queue.cc:260] T 00000000000000000000000000000000 P 7f8d2c54eaec417cbc734e62b470ab94 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 2.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3feaddadcc464a6daa4fb7b3dc17612c" member_type: VOTER last_known_addr { host: "127.30.246.254" port: 40109 } } peers { permanent_uuid: "7f8d2c54eaec417cbc734e62b470ab94" member_type: VOTER last_known_addr { host: "127.30.246.253" port: 36093 } } peers { permanent_uuid: "6d709ea9b98741149dfde6be4ec3747d" member_type: VOTER last_known_addr { host: "127.30.246.252" port: 39647 } }
02:11:19.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:19.307665 32442 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7f8d2c54eaec417cbc734e62b470ab94 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 2 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3feaddadcc464a6daa4fb7b3dc17612c" member_type: VOTER last_known_addr { host: "127.30.246.254" port: 40109 } } peers { permanent_uuid: "7f8d2c54eaec417cbc734e62b470ab94" member_type: VOTER last_known_addr { host: "127.30.246.253" port: 36093 } } peers { permanent_uuid: "6d709ea9b98741149dfde6be4ec3747d" member_type: VOTER last_known_addr { host: "127.30.246.252" port: 39647 } } }
02:11:19.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:19.308789 32442 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7f8d2c54eaec417cbc734e62b470ab94 [sys.catalog]: This master's current role is: FOLLOWER
02:11:19.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:19.309650 32438 sys_catalog.cc:565] T 00000000000000000000000000000000 P 7f8d2c54eaec417cbc734e62b470ab94 [sys.catalog]: configured and running, proceeding with master startup.
02:11:19.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:19.346501 32453 catalog_manager.cc:1269] Loaded cluster ID: 8be6e7d7c50a40e1b40d161833e26bd8
02:11:19.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:19.346889 32453 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 7f8d2c54eaec417cbc734e62b470ab94: loading cluster ID for follower catalog manager: success
02:11:19.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:19.358088 32453 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 7f8d2c54eaec417cbc734e62b470ab94: acquiring CA information for follower catalog manager: success
02:11:19.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:19.364533 32453 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 7f8d2c54eaec417cbc734e62b470ab94: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
02:11:19.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:19.352789 32363 master_runner.cc:186] Error getting master registration for 127.30.246.254:40109: Network error: Client connection negotiation failed: client connection to 127.30.246.254:40109: connect: Connection refused (error 111),
02:11:19.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:19.376760 32363 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
02:11:19.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:19.594532 32440 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:11:19.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:19.595192 32440 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:11:19.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:19.595920 32440 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:11:19.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:19.632478 32440 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
02:11:19.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:19.632773 32440 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
02:11:19.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:19.633164 32440 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:11:19.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:19.633484 32440 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
02:11:19.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:19.633777 32440 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
02:11:19.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:19.634066 32440 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
02:11:19.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:19.634287 32440 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:42087/jwks.json
02:11:19.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:19.675489 32440 master_runner.cc:386] Master server non-default flags:
02:11:19.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:11:19.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.246.212:35299
02:11:19.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:11:19.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:11:19.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-0/data
02:11:19.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-0/wal
02:11:19.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
02:11:19.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:11:19.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.30.246.254:40109,127.30.246.253:36093,127.30.246.252:39647
02:11:19.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/krb5kdc/kudu.keytab
02:11:19.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.30.246.254
02:11:19.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:11:19.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:11:19.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:11:19.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.246.254:40109
02:11:19.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:11:19.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:11:19.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:11:19.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42087/jwks.json
02:11:19.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:11:19.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
02:11:19.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
02:11:19.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:11:19.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:11:19.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-0/data/info.pb
02:11:19.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.246.254
02:11:19.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33217
02:11:19.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:11:19.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:11:19.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:11:19.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:11:19.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:11:19.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-0/logs
02:11:19.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:11:19.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:11:19.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:11:19.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:11:19.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:11:19.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 5688a9afc53de46b2b2f6847bc5217f38e926fe8
02:11:19.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:11:19.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 24 Oct 2025 01:43:20 UTC on 5fd53c4cbb9d
02:11:19.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8689
02:11:19.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:11:19.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:19.676941 32440 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:11:19.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:19.678045 32440 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:11:19.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:19.691133 32462 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:11:20.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:20.516570 32468 raft_consensus.cc:493] T 00000000000000000000000000000000 P 7f8d2c54eaec417cbc734e62b470ab94 [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:11:20.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:20.518127 32468 raft_consensus.cc:515] T 00000000000000000000000000000000 P 7f8d2c54eaec417cbc734e62b470ab94 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3feaddadcc464a6daa4fb7b3dc17612c" member_type: VOTER last_known_addr { host: "127.30.246.254" port: 40109 } } peers { permanent_uuid: "7f8d2c54eaec417cbc734e62b470ab94" member_type: VOTER last_known_addr { host: "127.30.246.253" port: 36093 } } peers { permanent_uuid: "6d709ea9b98741149dfde6be4ec3747d" member_type: VOTER last_known_addr { host: "127.30.246.252" port: 39647 } }
02:11:20.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:20.542968 32468 leader_election.cc:290] T 00000000000000000000000000000000 P 7f8d2c54eaec417cbc734e62b470ab94 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 3feaddadcc464a6daa4fb7b3dc17612c (127.30.246.254:40109), 6d709ea9b98741149dfde6be4ec3747d (127.30.246.252:39647)
02:11:20.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:20.557132 32388 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.30.246.254:40109: connect: Connection refused (error 111)
02:11:20.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:20.570300 32388 leader_election.cc:336] T 00000000000000000000000000000000 P 7f8d2c54eaec417cbc734e62b470ab94 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 3feaddadcc464a6daa4fb7b3dc17612c (127.30.246.254:40109): Network error: Client connection negotiation failed: client connection to 127.30.246.254:40109: connect: Connection refused (error 111)
02:11:20.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:20.575855 32389 leader_election.cc:336] T 00000000000000000000000000000000 P 7f8d2c54eaec417cbc734e62b470ab94 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 6d709ea9b98741149dfde6be4ec3747d (127.30.246.252:39647): Network error: Client connection negotiation failed: client connection to 127.30.246.252:39647: connect: Connection refused (error 111)
02:11:20.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:20.576722 32389 leader_election.cc:304] T 00000000000000000000000000000000 P 7f8d2c54eaec417cbc734e62b470ab94 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 7f8d2c54eaec417cbc734e62b470ab94; no voters: 3feaddadcc464a6daa4fb7b3dc17612c, 6d709ea9b98741149dfde6be4ec3747d
02:11:20.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:20.578984 32468 raft_consensus.cc:2749] T 00000000000000000000000000000000 P 7f8d2c54eaec417cbc734e62b470ab94 [term 2 FOLLOWER]: Leader pre-election lost for term 3. Reason: could not achieve majority
02:11:21.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:19.693066 32463 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:11:21.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:21.322849 32465 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:11:21.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:21.325611 32464 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1633 milliseconds
02:11:21.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:21.325724 32440 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:11:21.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:21.327643 32440 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:11:21.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:21.330942 32440 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:11:21.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:21.332458 32440 hybrid_clock.cc:648] HybridClock initialized: now 1761271881332360 us; error 109 us; skew 500 ppm
02:11:21.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:21.333635 32440 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:11:21.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 24 02:11:21 dist-test-slave-xv5f krb5kdc[31719](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761271881, etypes {rep=17 tkt=17 ses=17}, kudu/127.30.246.254@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:11:21.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:21.342070 32440 init.cc:377] Logged in from keytab as kudu/127.30.246.254@KRBTEST.COM (short username kudu)
02:11:21.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:21.354378 32440 webserver.cc:492] Webserver started at http://127.30.246.254:33217/ using document root <none> and password file <none>
02:11:21.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:21.355336 32440 fs_manager.cc:362] Metadata directory not provided
02:11:21.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:21.355535 32440 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:11:21.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:21.364292 32440 fs_manager.cc:714] Time spent opening directory manager: real 0.006s user 0.001s sys 0.004s
02:11:21.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:21.370862 32481 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:11:21.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:21.372206 32440 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.003s sys 0.000s
02:11:21.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:21.372520 32440 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-0/data,/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-0/wal
02:11:21.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3feaddadcc464a6daa4fb7b3dc17612c"
02:11:21.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-24 02:11:00 on dist-test-slave-xv5f"
02:11:21.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:21.374754 32440 fs_report.cc:389] FS layout report
02:11:21.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:11:21.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-0/wal
02:11:21.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-0/wal
02:11:21.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-0/data/data
02:11:21.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:11:21.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:11:21.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:11:21.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:11:21.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:11:21.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:11:21.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:11:21.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:11:21.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:11:21.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:11:21.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:11:21.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:21.436082 32440 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:11:21.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:21.442319 32440 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:11:21.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:21.442796 32440 kserver.cc:163] Server-wide thread pool size limit: 3276
02:11:21.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:21.538537 32440 rpc_server.cc:307] RPC server started. Bound to: 127.30.246.254:40109
02:11:21.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:21.538650 32532 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.30.246.254:40109 every 8 connection(s)
02:11:21.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:21.541926 32440 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-0/data/info.pb
02:11:21.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:21.548655 31707 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskOg0PFu/build/tsan/bin/kudu as pid 32440
02:11:21.549 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:11:21.549 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.30.246.252:39647
02:11:21.549 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
02:11:21.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:21.551363 31707 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskOg0PFu/build/tsan/bin/kudu
02:11:21.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskOg0PFu/build/tsan/bin/kudu
02:11:21.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-2/wal
02:11:21.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-2/data
02:11:21.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:11:21.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:11:21.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:11:21.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:11:21.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:11:21.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:11:21.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-2/logs
02:11:21.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-2/data/info.pb
02:11:21.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:11:21.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:11:21.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:11:21.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:11:21.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:11:21.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:11:21.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:11:21.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:11:21.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:11:21.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:11:21.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:11:21.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:11:21.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.246.252:39647
02:11:21.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.246.252
02:11:21.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37959
02:11:21.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/krb5kdc/kudu.keytab
02:11:21.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.30.246.252
02:11:21.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:11:21.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
02:11:21.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
02:11:21.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
02:11:21.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.246.212:35299
02:11:21.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:11:21.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:11:21.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:11:21.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:11:21.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.30.246.254:40109,127.30.246.253:36093,127.30.246.252:39647
02:11:21.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:11:21.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42087/jwks.json
02:11:21.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
02:11:21.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
02:11:21.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:21.555096 32533 sys_catalog.cc:263] Verifying existing consensus state
02:11:21.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:21.563634 32533 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3feaddadcc464a6daa4fb7b3dc17612c: Bootstrap starting.
02:11:21.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:21.619961 32533 log.cc:826] T 00000000000000000000000000000000 P 3feaddadcc464a6daa4fb7b3dc17612c: Log is configured to *not* fsync() on all Append() calls
02:11:21.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:21.643575 32533 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3feaddadcc464a6daa4fb7b3dc17612c: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
02:11:21.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:21.644661 32533 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3feaddadcc464a6daa4fb7b3dc17612c: Bootstrap complete.
02:11:21.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:21.682463 32533 raft_consensus.cc:359] T 00000000000000000000000000000000 P 3feaddadcc464a6daa4fb7b3dc17612c [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3feaddadcc464a6daa4fb7b3dc17612c" member_type: VOTER last_known_addr { host: "127.30.246.254" port: 40109 } } peers { permanent_uuid: "7f8d2c54eaec417cbc734e62b470ab94" member_type: VOTER last_known_addr { host: "127.30.246.253" port: 36093 } } peers { permanent_uuid: "6d709ea9b98741149dfde6be4ec3747d" member_type: VOTER last_known_addr { host: "127.30.246.252" port: 39647 } }
02:11:21.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:21.686220 32533 raft_consensus.cc:740] T 00000000000000000000000000000000 P 3feaddadcc464a6daa4fb7b3dc17612c [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3feaddadcc464a6daa4fb7b3dc17612c, State: Initialized, Role: FOLLOWER
02:11:21.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:21.687482 32533 consensus_queue.cc:260] T 00000000000000000000000000000000 P 3feaddadcc464a6daa4fb7b3dc17612c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 2.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3feaddadcc464a6daa4fb7b3dc17612c" member_type: VOTER last_known_addr { host: "127.30.246.254" port: 40109 } } peers { permanent_uuid: "7f8d2c54eaec417cbc734e62b470ab94" member_type: VOTER last_known_addr { host: "127.30.246.253" port: 36093 } } peers { permanent_uuid: "6d709ea9b98741149dfde6be4ec3747d" member_type: VOTER last_known_addr { host: "127.30.246.252" port: 39647 } }
02:11:21.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:21.691144 32537 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3feaddadcc464a6daa4fb7b3dc17612c [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 2 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3feaddadcc464a6daa4fb7b3dc17612c" member_type: VOTER last_known_addr { host: "127.30.246.254" port: 40109 } } peers { permanent_uuid: "7f8d2c54eaec417cbc734e62b470ab94" member_type: VOTER last_known_addr { host: "127.30.246.253" port: 36093 } } peers { permanent_uuid: "6d709ea9b98741149dfde6be4ec3747d" member_type: VOTER last_known_addr { host: "127.30.246.252" port: 39647 } } }
02:11:21.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:21.692147 32537 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3feaddadcc464a6daa4fb7b3dc17612c [sys.catalog]: This master's current role is: FOLLOWER
02:11:21.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:21.694213 32533 sys_catalog.cc:565] T 00000000000000000000000000000000 P 3feaddadcc464a6daa4fb7b3dc17612c [sys.catalog]: configured and running, proceeding with master startup.
02:11:21.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:21.758077 32548 catalog_manager.cc:1269] Loaded cluster ID: 8be6e7d7c50a40e1b40d161833e26bd8
02:11:21.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:21.758419 32548 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 3feaddadcc464a6daa4fb7b3dc17612c: loading cluster ID for follower catalog manager: success
02:11:21.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:21.765962 32548 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 3feaddadcc464a6daa4fb7b3dc17612c: acquiring CA information for follower catalog manager: success
02:11:21.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:21.771751 32548 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 3feaddadcc464a6daa4fb7b3dc17612c: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
02:11:21.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 24 02:11:21 dist-test-slave-xv5f krb5kdc[31719](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761271881, etypes {rep=17 tkt=17 ses=17}, kudu/127.30.246.254@KRBTEST.COM for kudu/127.30.246.254@KRBTEST.COM
02:11:21.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:21.852246 32440 master_runner.cc:186] Error getting master registration for 127.30.246.252:39647: Network error: Client connection negotiation failed: client connection to 127.30.246.252:39647: connect: Connection refused (error 111),
02:11:21.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:21.880404 32440 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
02:11:22.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:22.082008 32535 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:11:22.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:22.083060 32535 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:11:22.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:22.084324 32535 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:11:22.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:22.144240 32535 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
02:11:22.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:22.144586 32535 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
02:11:22.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:22.145059 32535 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:11:22.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:22.145414 32535 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
02:11:22.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:22.145747 32535 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
02:11:22.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:22.146075 32535 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
02:11:22.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:22.146324 32535 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:42087/jwks.json
02:11:22.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:22.215418 32535 master_runner.cc:386] Master server non-default flags:
02:11:22.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:11:22.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.246.212:35299
02:11:22.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:11:22.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:11:22.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-2/data
02:11:22.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-2/wal
02:11:22.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
02:11:22.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:11:22.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.30.246.254:40109,127.30.246.253:36093,127.30.246.252:39647
02:11:22.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/krb5kdc/kudu.keytab
02:11:22.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.30.246.252
02:11:22.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:11:22.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:11:22.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:11:22.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.246.252:39647
02:11:22.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:11:22.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:11:22.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:11:22.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42087/jwks.json
02:11:22.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:11:22.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
02:11:22.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
02:11:22.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:11:22.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:11:22.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-2/data/info.pb
02:11:22.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.246.252
02:11:22.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37959
02:11:22.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:11:22.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:11:22.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:11:22.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:11:22.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:11:22.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-2/logs
02:11:22.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:11:22.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:11:22.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:11:22.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:11:22.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:11:22.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 5688a9afc53de46b2b2f6847bc5217f38e926fe8
02:11:22.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:11:22.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 24 Oct 2025 01:43:20 UTC on 5fd53c4cbb9d
02:11:22.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8689
02:11:22.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:11:22.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:22.217147 32535 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:11:22.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:22.218560 32535 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:11:22.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:22.251925 32559 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:11:22.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:22.627995 32565 raft_consensus.cc:493] T 00000000000000000000000000000000 P 7f8d2c54eaec417cbc734e62b470ab94 [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:11:22.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:22.628472 32565 raft_consensus.cc:515] T 00000000000000000000000000000000 P 7f8d2c54eaec417cbc734e62b470ab94 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3feaddadcc464a6daa4fb7b3dc17612c" member_type: VOTER last_known_addr { host: "127.30.246.254" port: 40109 } } peers { permanent_uuid: "7f8d2c54eaec417cbc734e62b470ab94" member_type: VOTER last_known_addr { host: "127.30.246.253" port: 36093 } } peers { permanent_uuid: "6d709ea9b98741149dfde6be4ec3747d" member_type: VOTER last_known_addr { host: "127.30.246.252" port: 39647 } }
02:11:22.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:22.659919 32565 leader_election.cc:290] T 00000000000000000000000000000000 P 7f8d2c54eaec417cbc734e62b470ab94 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 3feaddadcc464a6daa4fb7b3dc17612c (127.30.246.254:40109), 6d709ea9b98741149dfde6be4ec3747d (127.30.246.252:39647)
02:11:22.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:22.686414 32389 leader_election.cc:336] T 00000000000000000000000000000000 P 7f8d2c54eaec417cbc734e62b470ab94 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 6d709ea9b98741149dfde6be4ec3747d (127.30.246.252:39647): Network error: Client connection negotiation failed: client connection to 127.30.246.252:39647: connect: Connection refused (error 111)
02:11:22.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:22.699702 32508 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "7f8d2c54eaec417cbc734e62b470ab94" candidate_term: 3 candidate_status { last_received { term: 2 index: 5 } } ignore_live_leader: false dest_uuid: "3feaddadcc464a6daa4fb7b3dc17612c" is_pre_election: true
02:11:22.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:22.700752 32508 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 3feaddadcc464a6daa4fb7b3dc17612c [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7f8d2c54eaec417cbc734e62b470ab94 in term 2.
02:11:22.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:22.703359 32388 leader_election.cc:304] T 00000000000000000000000000000000 P 7f8d2c54eaec417cbc734e62b470ab94 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 3feaddadcc464a6daa4fb7b3dc17612c, 7f8d2c54eaec417cbc734e62b470ab94; no voters: 6d709ea9b98741149dfde6be4ec3747d
02:11:22.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:22.704512 32565 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 7f8d2c54eaec417cbc734e62b470ab94 [term 2 FOLLOWER]: Leader pre-election won for term 3
02:11:22.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:22.704857 32565 raft_consensus.cc:493] T 00000000000000000000000000000000 P 7f8d2c54eaec417cbc734e62b470ab94 [term 2 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
02:11:22.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:22.705199 32565 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 7f8d2c54eaec417cbc734e62b470ab94 [term 2 FOLLOWER]: Advancing to term 3
02:11:22.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:22.719724 32565 raft_consensus.cc:515] T 00000000000000000000000000000000 P 7f8d2c54eaec417cbc734e62b470ab94 [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3feaddadcc464a6daa4fb7b3dc17612c" member_type: VOTER last_known_addr { host: "127.30.246.254" port: 40109 } } peers { permanent_uuid: "7f8d2c54eaec417cbc734e62b470ab94" member_type: VOTER last_known_addr { host: "127.30.246.253" port: 36093 } } peers { permanent_uuid: "6d709ea9b98741149dfde6be4ec3747d" member_type: VOTER last_known_addr { host: "127.30.246.252" port: 39647 } }
02:11:22.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:22.724048 32508 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "7f8d2c54eaec417cbc734e62b470ab94" candidate_term: 3 candidate_status { last_received { term: 2 index: 5 } } ignore_live_leader: false dest_uuid: "3feaddadcc464a6daa4fb7b3dc17612c"
02:11:22.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:22.724637 32508 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 3feaddadcc464a6daa4fb7b3dc17612c [term 2 FOLLOWER]: Advancing to term 3
02:11:22.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:22.755647 32389 leader_election.cc:336] T 00000000000000000000000000000000 P 7f8d2c54eaec417cbc734e62b470ab94 [CANDIDATE]: Term 3 election: RPC error from VoteRequest() call to peer 6d709ea9b98741149dfde6be4ec3747d (127.30.246.252:39647): Network error: Client connection negotiation failed: client connection to 127.30.246.252:39647: connect: Connection refused (error 111)
02:11:22.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:22.756738 32565 leader_election.cc:290] T 00000000000000000000000000000000 P 7f8d2c54eaec417cbc734e62b470ab94 [CANDIDATE]: Term 3 election: Requested vote from peers 3feaddadcc464a6daa4fb7b3dc17612c (127.30.246.254:40109), 6d709ea9b98741149dfde6be4ec3747d (127.30.246.252:39647)
02:11:22.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:22.760524 32508 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 3feaddadcc464a6daa4fb7b3dc17612c [term 3 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7f8d2c54eaec417cbc734e62b470ab94 in term 3.
02:11:22.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:22.761827 32388 leader_election.cc:304] T 00000000000000000000000000000000 P 7f8d2c54eaec417cbc734e62b470ab94 [CANDIDATE]: Term 3 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 3feaddadcc464a6daa4fb7b3dc17612c, 7f8d2c54eaec417cbc734e62b470ab94; no voters: 6d709ea9b98741149dfde6be4ec3747d
02:11:22.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:22.763670 32565 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 7f8d2c54eaec417cbc734e62b470ab94 [term 3 FOLLOWER]: Leader election won for term 3
02:11:22.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:22.783071 32565 raft_consensus.cc:697] T 00000000000000000000000000000000 P 7f8d2c54eaec417cbc734e62b470ab94 [term 3 LEADER]: Becoming Leader. State: Replica: 7f8d2c54eaec417cbc734e62b470ab94, State: Running, Role: LEADER
02:11:22.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:22.784045 32565 consensus_queue.cc:237] T 00000000000000000000000000000000 P 7f8d2c54eaec417cbc734e62b470ab94 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 2.5, Last appended by leader: 5, Current term: 3, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3feaddadcc464a6daa4fb7b3dc17612c" member_type: VOTER last_known_addr { host: "127.30.246.254" port: 40109 } } peers { permanent_uuid: "7f8d2c54eaec417cbc734e62b470ab94" member_type: VOTER last_known_addr { host: "127.30.246.253" port: 36093 } } peers { permanent_uuid: "6d709ea9b98741149dfde6be4ec3747d" member_type: VOTER last_known_addr { host: "127.30.246.252" port: 39647 } }
02:11:22.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:22.801306 32570 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7f8d2c54eaec417cbc734e62b470ab94 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 7f8d2c54eaec417cbc734e62b470ab94. Latest consensus state: current_term: 3 leader_uuid: "7f8d2c54eaec417cbc734e62b470ab94" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3feaddadcc464a6daa4fb7b3dc17612c" member_type: VOTER last_known_addr { host: "127.30.246.254" port: 40109 } } peers { permanent_uuid: "7f8d2c54eaec417cbc734e62b470ab94" member_type: VOTER last_known_addr { host: "127.30.246.253" port: 36093 } } peers { permanent_uuid: "6d709ea9b98741149dfde6be4ec3747d" member_type: VOTER last_known_addr { host: "127.30.246.252" port: 39647 } } }
02:11:22.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:22.802335 32570 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7f8d2c54eaec417cbc734e62b470ab94 [sys.catalog]: This master's current role is: LEADER
02:11:22.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:22.805212 32572 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
02:11:22.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:22.821350 32572 catalog_manager.cc:1494] Initializing Kudu cluster ID...
02:11:22.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:22.825188 32572 catalog_manager.cc:1269] Loaded cluster ID: 8be6e7d7c50a40e1b40d161833e26bd8
02:11:22.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:22.825495 32572 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
02:11:22.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:22.841046 32572 catalog_manager.cc:1514] Loading token signing keys...
02:11:22.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:22.853154 32572 catalog_manager.cc:6033] T 00000000000000000000000000000000 P 7f8d2c54eaec417cbc734e62b470ab94: Loaded TSK: 0
02:11:22.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:22.854911 32572 catalog_manager.cc:1524] Initializing in-progress tserver states...
02:11:23.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:23.215340 32389 consensus_peers.cc:597] T 00000000000000000000000000000000 P 7f8d2c54eaec417cbc734e62b470ab94 -> Peer 6d709ea9b98741149dfde6be4ec3747d (127.30.246.252:39647): Couldn't send request to peer 6d709ea9b98741149dfde6be4ec3747d. Status: Network error: Client connection negotiation failed: client connection to 127.30.246.252:39647: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
02:11:23.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:23.275207 32508 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 3feaddadcc464a6daa4fb7b3dc17612c [term 3 FOLLOWER]: Refusing update from remote peer 7f8d2c54eaec417cbc734e62b470ab94: Log matching property violated. Preceding OpId in replica: term: 2 index: 5. Preceding OpId from leader: term: 3 index: 6. (index mismatch)
02:11:23.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:23.277375 32570 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 7f8d2c54eaec417cbc734e62b470ab94 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3feaddadcc464a6daa4fb7b3dc17612c" member_type: VOTER last_known_addr { host: "127.30.246.254" port: 40109 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
02:11:23.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:23.320878 32573 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3feaddadcc464a6daa4fb7b3dc17612c [sys.catalog]: SysCatalogTable state changed. Reason: New leader 7f8d2c54eaec417cbc734e62b470ab94. Latest consensus state: current_term: 3 leader_uuid: "7f8d2c54eaec417cbc734e62b470ab94" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3feaddadcc464a6daa4fb7b3dc17612c" member_type: VOTER last_known_addr { host: "127.30.246.254" port: 40109 } } peers { permanent_uuid: "7f8d2c54eaec417cbc734e62b470ab94" member_type: VOTER last_known_addr { host: "127.30.246.253" port: 36093 } } peers { permanent_uuid: "6d709ea9b98741149dfde6be4ec3747d" member_type: VOTER last_known_addr { host: "127.30.246.252" port: 39647 } } }
02:11:23.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:23.321978 32573 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3feaddadcc464a6daa4fb7b3dc17612c [sys.catalog]: This master's current role is: FOLLOWER
02:11:23.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:23.349573 32573 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3feaddadcc464a6daa4fb7b3dc17612c [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "7f8d2c54eaec417cbc734e62b470ab94" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3feaddadcc464a6daa4fb7b3dc17612c" member_type: VOTER last_known_addr { host: "127.30.246.254" port: 40109 } } peers { permanent_uuid: "7f8d2c54eaec417cbc734e62b470ab94" member_type: VOTER last_known_addr { host: "127.30.246.253" port: 36093 } } peers { permanent_uuid: "6d709ea9b98741149dfde6be4ec3747d" member_type: VOTER last_known_addr { host: "127.30.246.252" port: 39647 } } }
02:11:23.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:23.350296 32573 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3feaddadcc464a6daa4fb7b3dc17612c [sys.catalog]: This master's current role is: FOLLOWER
02:11:23.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:23.368852 32570 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7f8d2c54eaec417cbc734e62b470ab94 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 3 leader_uuid: "7f8d2c54eaec417cbc734e62b470ab94" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3feaddadcc464a6daa4fb7b3dc17612c" member_type: VOTER last_known_addr { host: "127.30.246.254" port: 40109 } } peers { permanent_uuid: "7f8d2c54eaec417cbc734e62b470ab94" member_type: VOTER last_known_addr { host: "127.30.246.253" port: 36093 } } peers { permanent_uuid: "6d709ea9b98741149dfde6be4ec3747d" member_type: VOTER last_known_addr { host: "127.30.246.252" port: 39647 } } }
02:11:23.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:23.369616 32570 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7f8d2c54eaec417cbc734e62b470ab94 [sys.catalog]: This master's current role is: LEADER
02:11:24.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:24.476653 32363 master_runner.cc:186] Error getting master registration for 127.30.246.252:39647: Network error: Client connection negotiation failed: client connection to 127.30.246.252:39647: connect: Connection refused (error 111),
02:11:24.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:24.477133 32363 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
02:11:24.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:23.654098 32558 debug-util.cc:398] Leaking SignalData structure 0x7b0800027080 after lost signal to thread 32535
02:11:24.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:24.485108 32558 kernel_stack_watchdog.cc:198] Thread 32535 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 392ms:
02:11:24.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
02:11:24.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
02:11:24.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
02:11:24.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
02:11:24.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:22.253840 32560 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:11:24.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:24.492776 32561 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2232 milliseconds
02:11:24.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:24.493391 32535 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 2.234s user 0.005s sys 0.002s
02:11:24.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:24.493641 32535 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 2.234s user 0.005s sys 0.002s
02:11:24.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:24.495757 32535 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:11:24.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:24.495815 32563 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:11:24.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:24.497189 32535 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:11:24.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:24.500228 32535 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:11:24.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:24.501708 32535 hybrid_clock.cc:648] HybridClock initialized: now 1761271884501656 us; error 64 us; skew 500 ppm
02:11:24.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:24.502713 32535 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:11:24.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 24 02:11:24 dist-test-slave-xv5f krb5kdc[31719](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761271884, etypes {rep=17 tkt=17 ses=17}, kudu/127.30.246.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:11:24.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:24.509158 32535 init.cc:377] Logged in from keytab as kudu/127.30.246.252@KRBTEST.COM (short username kudu)
02:11:24.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:24.515406 32535 webserver.cc:492] Webserver started at http://127.30.246.252:37959/ using document root <none> and password file <none>
02:11:24.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:24.516710 32535 fs_manager.cc:362] Metadata directory not provided
02:11:24.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:24.516993 32535 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:11:24.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:24.528164 32535 fs_manager.cc:714] Time spent opening directory manager: real 0.007s user 0.008s sys 0.000s
02:11:24.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:24.534181 32592 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:11:24.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:24.535514 32535 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
02:11:24.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:24.535959 32535 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-2/data,/tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-2/wal
02:11:24.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6d709ea9b98741149dfde6be4ec3747d"
02:11:24.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-24 02:11:04 on dist-test-slave-xv5f"
02:11:24.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:24.538050 32535 fs_report.cc:389] FS layout report
02:11:24.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:11:24.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-2/wal
02:11:24.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-2/wal
02:11:24.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-2/data/data
02:11:24.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:11:24.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:11:24.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:11:24.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:11:24.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:11:24.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:11:24.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:11:24.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:11:24.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:11:24.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:11:24.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:11:24.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:24.597770 32535 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:11:24.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:24.608913 32535 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:11:24.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:24.609488 32535 kserver.cc:163] Server-wide thread pool size limit: 3276
02:11:24.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:24.700758 32535 rpc_server.cc:307] RPC server started. Bound to: 127.30.246.252:39647
02:11:24.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:24.700866 32643 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.30.246.252:39647 every 8 connection(s)
02:11:24.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:24.703946 32535 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/master-2/data/info.pb
02:11:24.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:24.711109 31707 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskOg0PFu/build/tsan/bin/kudu as pid 32535
02:11:24.711 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:11:24.712 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.30.246.253:36093(127.30.246.253:36093)] connecting to peer
02:11:24.715 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.30.246.253:36093(127.30.246.253:36093)] Successfully connected to peer
02:11:24.717 [DEBUG - kudu-nio-1] (SecurityContext.java:222) Refreshing Kerberos credentials...
02:11:24.717 [DEBUG - kudu-nio-1] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskOg0PFu/test-tmp/mini-kudu-cluster3201134858445564656/krb5kdc/krb5cc
02:11:24.719 [DEBUG - kudu-nio-1] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
02:11:24.719 [WARN - kudu-nio-1] (SecurityContext.java:231) Tried to refresh Kerberos credentials but was unable to re-login from ticket cache
02:11:24.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:24.719801 32644 sys_catalog.cc:263] Verifying existing consensus state
02:11:24.720 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.30.246.253:36093(127.30.246.253:36093)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
02:11:24.721 [DEBUG - main] (Connection.java:575) [peer master-127.30.246.254:40109(127.30.246.254:40109)] disconnecting while in state TERMINATED
02:11:24.722 [DEBUG - main] (Connection.java:575) [peer master-127.30.246.252:39647(127.30.246.252:39647)] disconnecting while in state TERMINATED
02:11:24.722 [DEBUG - main] (Connection.java:575) [peer master-127.30.246.253:36093(127.30.246.253:36093)] disconnecting while in state TERMINATED
02:11:24.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:24.722755 32586 negotiation.cc:337] Failed RPC negotiation. Trace:
02:11:24.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1024 02:11:24.714594 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:52940 (local address 127.30.246.253:36093)
02:11:24.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1024 02:11:24.715340 (+ 746us) server_negotiation.cc:207] Beginning negotiation
02:11:24.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1024 02:11:24.715368 (+ 28us) server_negotiation.cc:400] Waiting for connection header
02:11:24.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1024 02:11:24.715583 (+ 215us) server_negotiation.cc:408] Connection header received
02:11:24.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1024 02:11:24.716149 (+ 566us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
02:11:24.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1024 02:11:24.716179 (+ 30us) server_negotiation.cc:462] Received NEGOTIATE request from client
02:11:24.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1024 02:11:24.716489 (+ 310us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
02:11:24.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1024 02:11:24.716712 (+ 223us) server_negotiation.cc:227] Negotiated authn=SASL
02:11:24.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1024 02:11:24.721445 (+ 4733us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:52940: BlockingRecv error: recv got EOF from 127.0.0.1:52940 (error 108)
02:11:24.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":126}
02:11:24.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:24.727550 32644 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6d709ea9b98741149dfde6be4ec3747d: Bootstrap starting.
02:11:24.729 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2704) Releasing all remaining resources
02:11:24.734 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.30.246.253:36093(127.30.246.253:36093)] handling channelInactive
02:11:24.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:24.743229 31707 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskOg0PFu/build/tsan/bin/kudu with pid 32440
02:11:24.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:24.771791 31707 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskOg0PFu/build/tsan/bin/kudu with pid 32363
02:11:24.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:24.785017 32644 log.cc:826] T 00000000000000000000000000000000 P 6d709ea9b98741149dfde6be4ec3747d: Log is configured to *not* fsync() on all Append() calls
02:11:24.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:24.803344 32644 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6d709ea9b98741149dfde6be4ec3747d: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
02:11:24.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:24.804200 32644 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6d709ea9b98741149dfde6be4ec3747d: Bootstrap complete.
02:11:24.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:24.806326 31707 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskOg0PFu/build/tsan/bin/kudu with pid 32535
02:11:26.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-10-24T02:11:26Z chronyd exiting
02:11:26.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:26.087855 31552 debug-util.cc:398] Leaking SignalData structure 0x7b08000b3980 after lost signal to thread 31423
02:11:26.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251024 02:11:26.088820 31552 debug-util.cc:398] Leaking SignalData structure 0x7b08000adb40 after lost signal to thread 31555
02:11:27.069 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
02:11:27.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:27.071547 30972 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskOg0PFu/build/tsan/bin/kudu with pid 31256
02:11:27.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:27.108604 30972 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskOg0PFu/build/tsan/bin/kudu with pid 31418
02:11:27.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:27.137604 30972 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskOg0PFu/build/tsan/bin/kudu with pid 31566
02:11:27.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:27.161916 30972 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskOg0PFu/build/tsan/bin/kudu with pid 30984
02:11:27.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:27.194849 30972 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskOg0PFu/build/tsan/bin/kudu with pid 31054
02:11:27.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251024 02:11:27.225694 30972 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskOg0PFu/build/tsan/bin/kudu with pid 31129
02:11:28.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-10-24T02:11:28Z chronyd exiting
02:11:29.406 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testRenewAndReacquireKeberosCredentials: failed attempt 1
org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testRenewAndReacquireKeberosCredentials(TestSecurity.java:346) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.Negotiator.chooseAndInitializeSaslMech(Negotiator.java:565) ~[main/:?]
at org.apache.kudu.client.Negotiator.handleNegotiateResponse(Negotiator.java:433) ~[main/:?]
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:367) ~[main/:?]
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:94) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testRenewAndReacquireKeberosCredentials(TestSecurity.java:346)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:748)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.Negotiator.chooseAndInitializeSaslMech(Negotiator.java:565)
at org.apache.kudu.client.Negotiator.handleNegotiateResponse(Negotiator.java:433)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:367)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:94)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more