20:06:55.707 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
20:06:55.718 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taske7evVR/test-tmp
20:06:55.721 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
20:06:55.725 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taske7evVR/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
20:06:55.955 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster2568307083494362251"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
20:06:56.097 [DEBUG - main] (MiniKuduCluster.java:195) Response:
20:06:56.098 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
20:06:56.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:06:56.101781 10670 mini_oidc.cc:150] Starting JWKS server
20:06:56.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:06:56.105012 10670 webserver.cc:480] Webserver started at http://127.0.0.1:39057/ using document root <none> and password file <none>
20:06:56.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:06:56.105407 10670 mini_oidc.cc:186] Starting OIDC Discovery server
20:06:56.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:06:56.107146 10670 webserver.cc:480] Webserver started at http://127.0.0.1:37681/ using document root <none> and password file <none>
20:06:56.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-09-05T20:06:56Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
20:06:56.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-09-05T20:06:56Z Disabled control of system clock
20:06:56.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:06:56.165810 10670 external_mini_cluster.cc:1366] Running /tmp/dist-test-taske7evVR/build/tsan/bin/kudu
20:06:56.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taske7evVR/build/tsan/bin/kudu
20:06:56.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster2568307083494362251/master-0/wal
20:06:56.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster2568307083494362251/master-0/data
20:06:56.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:06:56.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:06:56.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:06:56.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:06:56.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:06:56.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:06:56.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster2568307083494362251/master-0/logs
20:06:56.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster2568307083494362251/master-0/data/info.pb
20:06:56.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:06:56.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:06:56.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:06:56.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:06:56.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:06:56.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:06:56.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:06:56.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:06:56.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:06:56.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:06:56.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:06:56.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:06:56.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.107.190:42101
20:06:56.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.107.190
20:06:56.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:06:56.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.107.148:41887
20:06:56.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:06:56.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:06:56.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:06:56.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:06:56.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.107.190:42101,127.10.107.189:36917,127.10.107.188:34039
20:06:56.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:06:56.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39057/jwks.json with env {}
20:06:56.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:06:56.468652 10682 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:06:56.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:06:56.469199 10682 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:06:56.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:06:56.469738 10682 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:06:56.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:06:56.500018 10682 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
20:06:56.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:06:56.500310 10682 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:06:56.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:06:56.500506 10682 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
20:06:56.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:06:56.500723 10682 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
20:06:56.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:06:56.500991 10682 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
20:06:56.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:06:56.501139 10682 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:39057/jwks.json
20:06:56.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:06:56.538398 10682 master_runner.cc:386] Master server non-default flags:
20:06:56.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:06:56.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.107.148:41887
20:06:56.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:06:56.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:06:56.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster2568307083494362251/master-0/data
20:06:56.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster2568307083494362251/master-0/wal
20:06:56.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:06:56.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.107.190:42101,127.10.107.189:36917,127.10.107.188:34039
20:06:56.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:06:56.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:06:56.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:06:56.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.107.190:42101
20:06:56.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:06:56.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:06:56.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:06:56.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39057/jwks.json
20:06:56.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:06:56.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:06:56.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster2568307083494362251/master-0/data/info.pb
20:06:56.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.107.190
20:06:56.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:06:56.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:06:56.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:06:56.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:06:56.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:06:56.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:06:56.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster2568307083494362251/master-0/logs
20:06:56.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:06:56.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:06:56.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:06:56.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:06:56.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:06:56.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision e2e116a9cb215058aad161061214d11c617cedac
20:06:56.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:06:56.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 05 Sep 2025 19:43:22 UTC on 5fd53c4cbb9d
20:06:56.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7917
20:06:56.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:06:56.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:06:56.539959 10682 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:06:56.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:06:56.541844 10682 file_cache.cc:492] Constructed file cache file cache with capacity 419430
20:06:56.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:06:56.553210 10688 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:06:58.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:06:57.956977 10687 debug-util.cc:398] Leaking SignalData structure 0x7b0800034ec0 after lost signal to thread 10682
20:06:58.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:06:58.296865 10687 kernel_stack_watchdog.cc:198] Thread 10682 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 400ms:
20:06:58.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
20:06:58.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
20:06:58.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
20:06:58.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
20:06:58.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:06:56.554342 10689 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:06:58.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:06:58.298098 10682 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.744s user 0.729s sys 1.014s
20:06:58.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:06:58.298435 10682 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.745s user 0.729s sys 1.014s
20:06:58.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:06:58.299978 10691 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:06:58.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:06:58.302397 10690 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1743 milliseconds
20:06:58.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:06:58.302412 10682 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:06:58.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:06:58.303561 10682 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:06:58.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:06:58.305871 10682 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:06:58.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:06:58.307178 10682 hybrid_clock.cc:648] HybridClock initialized: now 1757102818307150 us; error 36 us; skew 500 ppm
20:06:58.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:06:58.307907 10682 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:06:58.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:06:58.313484 10682 webserver.cc:480] Webserver started at http://127.10.107.190:42735/ using document root <none> and password file <none>
20:06:58.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:06:58.314311 10682 fs_manager.cc:362] Metadata directory not provided
20:06:58.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:06:58.314479 10682 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:06:58.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:06:58.314965 10682 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
20:06:58.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:06:58.319322 10682 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster2568307083494362251/master-0/data/instance:
20:06:58.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d51056216da443c0bd535a9d772c37c2"
20:06:58.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-05 20:06:58 on dist-test-slave-x5g2"
20:06:58.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:06:58.320428 10682 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster2568307083494362251/master-0/wal/instance:
20:06:58.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d51056216da443c0bd535a9d772c37c2"
20:06:58.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-05 20:06:58 on dist-test-slave-x5g2"
20:06:58.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:06:58.327704 10682 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.009s sys 0.000s
20:06:58.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:06:58.333083 10698 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:06:58.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:06:58.334012 10682 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
20:06:58.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:06:58.334316 10682 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster2568307083494362251/master-0/data,/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster2568307083494362251/master-0/wal
20:06:58.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d51056216da443c0bd535a9d772c37c2"
20:06:58.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-05 20:06:58 on dist-test-slave-x5g2"
20:06:58.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:06:58.334689 10682 fs_report.cc:389] FS layout report
20:06:58.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:06:58.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster2568307083494362251/master-0/wal
20:06:58.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster2568307083494362251/master-0/wal
20:06:58.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster2568307083494362251/master-0/data/data
20:06:58.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:06:58.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:06:58.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:06:58.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:06:58.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:06:58.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:06:58.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:06:58.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:06:58.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:06:58.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:06:58.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:06:58.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:06:58.385196 10682 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:06:58.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:06:58.386688 10682 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:06:58.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:06:58.387130 10682 kserver.cc:163] Server-wide thread pool size limit: 3276
20:06:58.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:06:58.457722 10682 rpc_server.cc:307] RPC server started. Bound to: 127.10.107.190:42101
20:06:58.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:06:58.457811 10749 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.10.107.190:42101 every 8 connection(s)
20:06:58.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:06:58.460656 10682 server_base.cc:1179] Dumped server information to /tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster2568307083494362251/master-0/data/info.pb
20:06:58.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:06:58.466467 10750 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
20:06:58.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:06:58.469590 10670 external_mini_cluster.cc:1428] Started /tmp/dist-test-taske7evVR/build/tsan/bin/kudu as pid 10682
20:06:58.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:06:58.470140 10670 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster2568307083494362251/master-0/wal/instance
20:06:58.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:06:58.475524 10670 external_mini_cluster.cc:1366] Running /tmp/dist-test-taske7evVR/build/tsan/bin/kudu
20:06:58.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taske7evVR/build/tsan/bin/kudu
20:06:58.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster2568307083494362251/master-1/wal
20:06:58.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster2568307083494362251/master-1/data
20:06:58.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:06:58.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:06:58.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:06:58.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:06:58.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:06:58.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:06:58.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster2568307083494362251/master-1/logs
20:06:58.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster2568307083494362251/master-1/data/info.pb
20:06:58.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:06:58.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:06:58.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:06:58.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:06:58.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:06:58.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:06:58.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:06:58.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:06:58.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:06:58.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:06:58.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:06:58.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:06:58.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.107.189:36917
20:06:58.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.107.189
20:06:58.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:06:58.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.107.148:41887
20:06:58.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:06:58.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:06:58.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:06:58.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:06:58.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.107.190:42101,127.10.107.189:36917,127.10.107.188:34039
20:06:58.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:06:58.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39057/jwks.json with env {}
20:06:58.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:06:58.482304 10750 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.107.190" port: 42101 } has no permanent_uuid. Determining permanent_uuid...
20:06:58.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:06:58.506068 10750 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.107.189" port: 36917 } has no permanent_uuid. Determining permanent_uuid...
20:06:58.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:06:58.508534 10700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.10.107.189:36917: connect: Connection refused (error 111)
20:06:58.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:06:58.512856 10750 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.10.107.189:36917: Network error: Client connection negotiation failed: client connection to 127.10.107.189:36917: connect: Connection refused (error 111)
20:06:58.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:06:58.562281 10750 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.107.189" port: 36917 } attempt: 1
20:06:58.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:06:58.566239 10750 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.10.107.189:36917: Network error: Client connection negotiation failed: client connection to 127.10.107.189:36917: connect: Connection refused (error 111)
20:06:58.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:06:58.634725 10750 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.107.189" port: 36917 } attempt: 2
20:06:58.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:06:58.639170 10750 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.10.107.189:36917: Network error: Client connection negotiation failed: client connection to 127.10.107.189:36917: connect: Connection refused (error 111)
20:06:58.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:06:58.730568 10750 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.107.189" port: 36917 } attempt: 3
20:06:58.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:06:58.734403 10750 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.10.107.189:36917: Network error: Client connection negotiation failed: client connection to 127.10.107.189:36917: connect: Connection refused (error 111)
20:06:58.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:06:58.788388 10752 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:06:58.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:06:58.788945 10752 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:06:58.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:06:58.789408 10752 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:06:58.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:06:58.820909 10752 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
20:06:58.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:06:58.821194 10752 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:06:58.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:06:58.821393 10752 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
20:06:58.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:06:58.821645 10752 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
20:06:58.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:06:58.821847 10752 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
20:06:58.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:06:58.822077 10752 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:39057/jwks.json
20:06:58.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:06:58.857892 10752 master_runner.cc:386] Master server non-default flags:
20:06:58.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:06:58.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.107.148:41887
20:06:58.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:06:58.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:06:58.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster2568307083494362251/master-1/data
20:06:58.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster2568307083494362251/master-1/wal
20:06:58.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:06:58.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.107.190:42101,127.10.107.189:36917,127.10.107.188:34039
20:06:58.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:06:58.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:06:58.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:06:58.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.107.189:36917
20:06:58.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:06:58.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:06:58.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:06:58.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39057/jwks.json
20:06:58.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:06:58.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:06:58.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster2568307083494362251/master-1/data/info.pb
20:06:58.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.107.189
20:06:58.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:06:58.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:06:58.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:06:58.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:06:58.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:06:58.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:06:58.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster2568307083494362251/master-1/logs
20:06:58.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:06:58.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:06:58.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:06:58.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:06:58.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:06:58.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision e2e116a9cb215058aad161061214d11c617cedac
20:06:58.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:06:58.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 05 Sep 2025 19:43:22 UTC on 5fd53c4cbb9d
20:06:58.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7917
20:06:58.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:06:58.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:06:58.859316 10752 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:06:58.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:06:58.861572 10752 file_cache.cc:492] Constructed file cache file cache with capacity 419430
20:06:58.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:06:58.872560 10761 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:06:58.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:06:58.877756 10750 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.107.189" port: 36917 } attempt: 4
20:06:58.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:06:58.881359 10750 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.10.107.189:36917: Network error: Client connection negotiation failed: client connection to 127.10.107.189:36917: connect: Connection refused (error 111)
20:06:59.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:06:59.180876 10750 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.107.189" port: 36917 } attempt: 5
20:06:59.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:06:59.185041 10750 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.10.107.189:36917: Network error: Client connection negotiation failed: client connection to 127.10.107.189:36917: connect: Connection refused (error 111)
20:06:59.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:06:59.732465 10750 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.107.189" port: 36917 } attempt: 6
20:06:59.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:06:59.743021 10750 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.10.107.189:36917: Network error: Client connection negotiation failed: client connection to 127.10.107.189:36917: connect: Connection refused (error 111)
20:07:00.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:06:58.874794 10762 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:07:00.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:00.067178 10764 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:07:00.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:00.069101 10763 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
20:07:00.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:00.069237 10752 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:07:00.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:00.070640 10752 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:07:00.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:00.073851 10752 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:07:00.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:00.075274 10752 hybrid_clock.cc:648] HybridClock initialized: now 1757102820075227 us; error 57 us; skew 500 ppm
20:07:00.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:00.076144 10752 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:07:00.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:00.082923 10752 webserver.cc:480] Webserver started at http://127.10.107.189:44973/ using document root <none> and password file <none>
20:07:00.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:00.083798 10752 fs_manager.cc:362] Metadata directory not provided
20:07:00.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:00.083981 10752 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:07:00.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:00.084378 10752 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
20:07:00.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:00.088538 10752 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster2568307083494362251/master-1/data/instance:
20:07:00.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "86124f48110c470db8b724135148aaf2"
20:07:00.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-05 20:07:00 on dist-test-slave-x5g2"
20:07:00.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:00.089594 10752 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster2568307083494362251/master-1/wal/instance:
20:07:00.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "86124f48110c470db8b724135148aaf2"
20:07:00.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-05 20:07:00 on dist-test-slave-x5g2"
20:07:00.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:00.098264 10752 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.007s sys 0.001s
20:07:00.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:00.103709 10773 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:07:00.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:00.104918 10752 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
20:07:00.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:00.105212 10752 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster2568307083494362251/master-1/data,/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster2568307083494362251/master-1/wal
20:07:00.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "86124f48110c470db8b724135148aaf2"
20:07:00.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-05 20:07:00 on dist-test-slave-x5g2"
20:07:00.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:00.105579 10752 fs_report.cc:389] FS layout report
20:07:00.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:07:00.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster2568307083494362251/master-1/wal
20:07:00.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster2568307083494362251/master-1/wal
20:07:00.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster2568307083494362251/master-1/data/data
20:07:00.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:07:00.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:07:00.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:07:00.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:07:00.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:07:00.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:07:00.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:07:00.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:07:00.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:07:00.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:07:00.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:07:00.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:00.155530 10752 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:07:00.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:00.157145 10752 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:07:00.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:00.157648 10752 kserver.cc:163] Server-wide thread pool size limit: 3276
20:07:00.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:00.227792 10752 rpc_server.cc:307] RPC server started. Bound to: 127.10.107.189:36917
20:07:00.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:00.227860 10824 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.10.107.189:36917 every 8 connection(s)
20:07:00.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:00.230690 10752 server_base.cc:1179] Dumped server information to /tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster2568307083494362251/master-1/data/info.pb
20:07:00.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:00.235131 10670 external_mini_cluster.cc:1428] Started /tmp/dist-test-taske7evVR/build/tsan/bin/kudu as pid 10752
20:07:00.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:00.235620 10670 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster2568307083494362251/master-1/wal/instance
20:07:00.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:00.237310 10825 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
20:07:00.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:00.238588 10670 external_mini_cluster.cc:1366] Running /tmp/dist-test-taske7evVR/build/tsan/bin/kudu
20:07:00.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taske7evVR/build/tsan/bin/kudu
20:07:00.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster2568307083494362251/master-2/wal
20:07:00.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster2568307083494362251/master-2/data
20:07:00.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:07:00.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:07:00.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:07:00.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:07:00.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:07:00.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:07:00.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster2568307083494362251/master-2/logs
20:07:00.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster2568307083494362251/master-2/data/info.pb
20:07:00.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:07:00.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:07:00.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:07:00.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:07:00.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:07:00.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:07:00.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:07:00.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:07:00.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:07:00.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:07:00.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:07:00.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:07:00.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.107.188:34039
20:07:00.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.107.188
20:07:00.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:07:00.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.107.148:41887
20:07:00.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:07:00.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:07:00.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:07:00.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:07:00.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.107.190:42101,127.10.107.189:36917,127.10.107.188:34039
20:07:00.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:07:00.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39057/jwks.json with env {}
20:07:00.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:00.250545 10825 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.107.190" port: 42101 } has no permanent_uuid. Determining permanent_uuid...
20:07:00.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:00.279258 10825 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.107.189" port: 36917 } has no permanent_uuid. Determining permanent_uuid...
20:07:00.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:00.294673 10825 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.107.188" port: 34039 } has no permanent_uuid. Determining permanent_uuid...
20:07:00.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:00.297297 10777 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.10.107.188:34039: connect: Connection refused (error 111)
20:07:00.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:00.301460 10825 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.10.107.188:34039: Network error: Client connection negotiation failed: client connection to 127.10.107.188:34039: connect: Connection refused (error 111)
20:07:00.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:00.350942 10825 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.107.188" port: 34039 } attempt: 1
20:07:00.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:00.355109 10825 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.10.107.188:34039: Network error: Client connection negotiation failed: client connection to 127.10.107.188:34039: connect: Connection refused (error 111)
20:07:00.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:00.423465 10825 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.107.188" port: 34039 } attempt: 2
20:07:00.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:00.427779 10825 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.10.107.188:34039: Network error: Client connection negotiation failed: client connection to 127.10.107.188:34039: connect: Connection refused (error 111)
20:07:00.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:00.519165 10825 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.107.188" port: 34039 } attempt: 3
20:07:00.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:00.523252 10825 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.10.107.188:34039: Network error: Client connection negotiation failed: client connection to 127.10.107.188:34039: connect: Connection refused (error 111)
20:07:00.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:00.562279 10827 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:07:00.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:00.562877 10827 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:07:00.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:00.563472 10827 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:07:00.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:00.596252 10827 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
20:07:00.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:00.596555 10827 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:07:00.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:00.596752 10827 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
20:07:00.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:00.596975 10827 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
20:07:00.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:00.597224 10827 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
20:07:00.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:00.597501 10827 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:39057/jwks.json
20:07:00.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:00.634776 10827 master_runner.cc:386] Master server non-default flags:
20:07:00.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:07:00.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.107.148:41887
20:07:00.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:07:00.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:07:00.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster2568307083494362251/master-2/data
20:07:00.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster2568307083494362251/master-2/wal
20:07:00.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:07:00.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.107.190:42101,127.10.107.189:36917,127.10.107.188:34039
20:07:00.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:07:00.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:07:00.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:07:00.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.107.188:34039
20:07:00.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:07:00.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:07:00.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:07:00.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39057/jwks.json
20:07:00.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:07:00.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:07:00.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster2568307083494362251/master-2/data/info.pb
20:07:00.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.107.188
20:07:00.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:07:00.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:07:00.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:07:00.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:07:00.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:07:00.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:07:00.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster2568307083494362251/master-2/logs
20:07:00.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:07:00.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:07:00.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:07:00.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:07:00.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:07:00.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision e2e116a9cb215058aad161061214d11c617cedac
20:07:00.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:07:00.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 05 Sep 2025 19:43:22 UTC on 5fd53c4cbb9d
20:07:00.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7917
20:07:00.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:07:00.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:00.636111 10827 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:07:00.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:00.637992 10827 file_cache.cc:492] Constructed file cache file cache with capacity 419430
20:07:00.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:00.651396 10838 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:07:00.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:00.666630 10825 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.107.188" port: 34039 } attempt: 4
20:07:00.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:00.673976 10825 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.10.107.188:34039: Network error: Client connection negotiation failed: client connection to 127.10.107.188:34039: connect: Connection refused (error 111)
20:07:00.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:00.803723 10750 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.107.189" port: 36917 } attempt: 7
20:07:00.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:00.859979 10750 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.107.188" port: 34039 } has no permanent_uuid. Determining permanent_uuid...
20:07:00.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:00.880210 10750 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.10.107.188:34039: Network error: Client connection negotiation failed: client connection to 127.10.107.188:34039: connect: Connection refused (error 111)
20:07:00.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:00.938830 10750 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.107.188" port: 34039 } attempt: 1
20:07:00.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:00.946978 10750 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.10.107.188:34039: Network error: Client connection negotiation failed: client connection to 127.10.107.188:34039: connect: Connection refused (error 111)
20:07:00.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:00.973544 10825 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.107.188" port: 34039 } attempt: 5
20:07:00.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:00.981874 10825 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.10.107.188:34039: Network error: Client connection negotiation failed: client connection to 127.10.107.188:34039: connect: Connection refused (error 111)
20:07:01.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:01.028574 10750 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.107.188" port: 34039 } attempt: 2
20:07:01.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:01.036609 10750 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.10.107.188:34039: Network error: Client connection negotiation failed: client connection to 127.10.107.188:34039: connect: Connection refused (error 111)
20:07:01.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:01.122148 10750 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.107.188" port: 34039 } attempt: 3
20:07:01.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:01.129904 10750 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.10.107.188:34039: Network error: Client connection negotiation failed: client connection to 127.10.107.188:34039: connect: Connection refused (error 111)
20:07:01.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:01.273792 10750 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.107.188" port: 34039 } attempt: 4
20:07:01.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:01.281729 10750 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.10.107.188:34039: Network error: Client connection negotiation failed: client connection to 127.10.107.188:34039: connect: Connection refused (error 111)
20:07:01.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:01.529537 10825 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.107.188" port: 34039 } attempt: 6
20:07:01.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:01.561380 10825 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.10.107.188:34039: Network error: Client connection negotiation failed: client connection to 127.10.107.188:34039: connect: Connection refused (error 111)
20:07:01.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:01.565331 10750 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.107.188" port: 34039 } attempt: 5
20:07:01.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:01.573732 10750 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.10.107.188:34039: Network error: Client connection negotiation failed: client connection to 127.10.107.188:34039: connect: Connection refused (error 111)
20:07:02.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:02.126312 10750 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.107.188" port: 34039 } attempt: 6
20:07:02.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:02.162778 10750 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.10.107.188:34039: Network error: Client connection negotiation failed: client connection to 127.10.107.188:34039: connect: Connection refused (error 111)
20:07:02.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:02.053663 10837 debug-util.cc:398] Leaking SignalData structure 0x7b0800034ec0 after lost signal to thread 10827
20:07:02.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:02.430373 10837 kernel_stack_watchdog.cc:198] Thread 10827 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 398ms:
20:07:02.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
20:07:02.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
20:07:02.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
20:07:02.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
20:07:02.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:02.431107 10827 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.779s user 0.621s sys 1.151s
20:07:02.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:02.431468 10827 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.779s user 0.621s sys 1.151s
20:07:02.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:00.651831 10839 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:07:02.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:02.433914 10841 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:07:02.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:02.436643 10840 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1780 milliseconds
20:07:02.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:02.436712 10827 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:07:02.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:02.437940 10827 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:07:02.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:02.440307 10827 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:07:02.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:02.441641 10827 hybrid_clock.cc:648] HybridClock initialized: now 1757102822441588 us; error 59 us; skew 500 ppm
20:07:02.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:02.442411 10827 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:07:02.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:02.448098 10827 webserver.cc:480] Webserver started at http://127.10.107.188:34309/ using document root <none> and password file <none>
20:07:02.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:02.448901 10827 fs_manager.cc:362] Metadata directory not provided
20:07:02.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:02.449062 10827 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:07:02.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:02.449402 10827 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
20:07:02.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:02.453626 10827 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster2568307083494362251/master-2/data/instance:
20:07:02.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ce27da99bbe3488ca876547f04030d92"
20:07:02.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-05 20:07:02 on dist-test-slave-x5g2"
20:07:02.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:02.454587 10827 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster2568307083494362251/master-2/wal/instance:
20:07:02.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ce27da99bbe3488ca876547f04030d92"
20:07:02.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-05 20:07:02 on dist-test-slave-x5g2"
20:07:02.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:02.462669 10827 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.006s sys 0.002s
20:07:02.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:02.469235 10855 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:07:02.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:02.470366 10827 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.001s sys 0.002s
20:07:02.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:02.470651 10827 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster2568307083494362251/master-2/data,/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster2568307083494362251/master-2/wal
20:07:02.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ce27da99bbe3488ca876547f04030d92"
20:07:02.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-05 20:07:02 on dist-test-slave-x5g2"
20:07:02.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:02.470932 10827 fs_report.cc:389] FS layout report
20:07:02.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:07:02.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster2568307083494362251/master-2/wal
20:07:02.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster2568307083494362251/master-2/wal
20:07:02.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster2568307083494362251/master-2/data/data
20:07:02.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:07:02.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:07:02.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:07:02.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:07:02.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:07:02.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:07:02.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:07:02.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:07:02.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:07:02.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:07:02.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:07:02.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:02.532478 10827 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:07:02.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:02.534045 10827 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:07:02.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:02.534426 10827 kserver.cc:163] Server-wide thread pool size limit: 3276
20:07:02.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:02.601729 10827 rpc_server.cc:307] RPC server started. Bound to: 127.10.107.188:34039
20:07:02.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:02.601775 10906 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.10.107.188:34039 every 8 connection(s)
20:07:02.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:02.604737 10827 server_base.cc:1179] Dumped server information to /tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster2568307083494362251/master-2/data/info.pb
20:07:02.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:02.609117 10670 external_mini_cluster.cc:1428] Started /tmp/dist-test-taske7evVR/build/tsan/bin/kudu as pid 10827
20:07:02.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:02.609458 10670 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster2568307083494362251/master-2/wal/instance
20:07:02.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:02.610460 10907 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
20:07:02.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:02.622076 10825 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.107.188" port: 34039 } attempt: 7
20:07:02.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:02.624143 10907 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.107.190" port: 42101 } has no permanent_uuid. Determining permanent_uuid...
20:07:02.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:02.654379 10907 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.107.189" port: 36917 } has no permanent_uuid. Determining permanent_uuid...
20:07:02.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:02.666308 10907 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.107.188" port: 34039 } has no permanent_uuid. Determining permanent_uuid...
20:07:02.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:02.671674 10825 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 86124f48110c470db8b724135148aaf2: Bootstrap starting.
20:07:02.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:02.679760 10825 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 86124f48110c470db8b724135148aaf2: Neither blocks nor log segments found. Creating new log.
20:07:02.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:02.682134 10825 log.cc:826] T 00000000000000000000000000000000 P 86124f48110c470db8b724135148aaf2: Log is configured to *not* fsync() on all Append() calls
20:07:02.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:02.687112 10825 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 86124f48110c470db8b724135148aaf2: No bootstrap required, opened a new log
20:07:02.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:02.687671 10907 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ce27da99bbe3488ca876547f04030d92: Bootstrap starting.
20:07:02.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:02.693866 10907 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P ce27da99bbe3488ca876547f04030d92: Neither blocks nor log segments found. Creating new log.
20:07:02.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:02.695935 10907 log.cc:826] T 00000000000000000000000000000000 P ce27da99bbe3488ca876547f04030d92: Log is configured to *not* fsync() on all Append() calls
20:07:02.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:02.700613 10907 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ce27da99bbe3488ca876547f04030d92: No bootstrap required, opened a new log
20:07:02.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:02.707064 10825 raft_consensus.cc:357] T 00000000000000000000000000000000 P 86124f48110c470db8b724135148aaf2 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d51056216da443c0bd535a9d772c37c2" member_type: VOTER last_known_addr { host: "127.10.107.190" port: 42101 } } peers { permanent_uuid: "86124f48110c470db8b724135148aaf2" member_type: VOTER last_known_addr { host: "127.10.107.189" port: 36917 } } peers { permanent_uuid: "ce27da99bbe3488ca876547f04030d92" member_type: VOTER last_known_addr { host: "127.10.107.188" port: 34039 } }
20:07:02.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:02.707798 10825 raft_consensus.cc:383] T 00000000000000000000000000000000 P 86124f48110c470db8b724135148aaf2 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:07:02.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:02.708104 10825 raft_consensus.cc:738] T 00000000000000000000000000000000 P 86124f48110c470db8b724135148aaf2 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 86124f48110c470db8b724135148aaf2, State: Initialized, Role: FOLLOWER
20:07:02.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:02.708964 10825 consensus_queue.cc:260] T 00000000000000000000000000000000 P 86124f48110c470db8b724135148aaf2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d51056216da443c0bd535a9d772c37c2" member_type: VOTER last_known_addr { host: "127.10.107.190" port: 42101 } } peers { permanent_uuid: "86124f48110c470db8b724135148aaf2" member_type: VOTER last_known_addr { host: "127.10.107.189" port: 36917 } } peers { permanent_uuid: "ce27da99bbe3488ca876547f04030d92" member_type: VOTER last_known_addr { host: "127.10.107.188" port: 34039 } }
20:07:02.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:02.711628 10919 sys_catalog.cc:455] T 00000000000000000000000000000000 P 86124f48110c470db8b724135148aaf2 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d51056216da443c0bd535a9d772c37c2" member_type: VOTER last_known_addr { host: "127.10.107.190" port: 42101 } } peers { permanent_uuid: "86124f48110c470db8b724135148aaf2" member_type: VOTER last_known_addr { host: "127.10.107.189" port: 36917 } } peers { permanent_uuid: "ce27da99bbe3488ca876547f04030d92" member_type: VOTER last_known_addr { host: "127.10.107.188" port: 34039 } } }
20:07:02.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:02.712407 10919 sys_catalog.cc:458] T 00000000000000000000000000000000 P 86124f48110c470db8b724135148aaf2 [sys.catalog]: This master's current role is: FOLLOWER
20:07:02.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:02.713395 10825 sys_catalog.cc:564] T 00000000000000000000000000000000 P 86124f48110c470db8b724135148aaf2 [sys.catalog]: configured and running, proceeding with master startup.
20:07:02.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:02.721642 10907 raft_consensus.cc:357] T 00000000000000000000000000000000 P ce27da99bbe3488ca876547f04030d92 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d51056216da443c0bd535a9d772c37c2" member_type: VOTER last_known_addr { host: "127.10.107.190" port: 42101 } } peers { permanent_uuid: "86124f48110c470db8b724135148aaf2" member_type: VOTER last_known_addr { host: "127.10.107.189" port: 36917 } } peers { permanent_uuid: "ce27da99bbe3488ca876547f04030d92" member_type: VOTER last_known_addr { host: "127.10.107.188" port: 34039 } }
20:07:02.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:02.722626 10907 raft_consensus.cc:383] T 00000000000000000000000000000000 P ce27da99bbe3488ca876547f04030d92 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:07:02.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:02.722884 10907 raft_consensus.cc:738] T 00000000000000000000000000000000 P ce27da99bbe3488ca876547f04030d92 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ce27da99bbe3488ca876547f04030d92, State: Initialized, Role: FOLLOWER
20:07:02.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:02.723578 10907 consensus_queue.cc:260] T 00000000000000000000000000000000 P ce27da99bbe3488ca876547f04030d92 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d51056216da443c0bd535a9d772c37c2" member_type: VOTER last_known_addr { host: "127.10.107.190" port: 42101 } } peers { permanent_uuid: "86124f48110c470db8b724135148aaf2" member_type: VOTER last_known_addr { host: "127.10.107.189" port: 36917 } } peers { permanent_uuid: "ce27da99bbe3488ca876547f04030d92" member_type: VOTER last_known_addr { host: "127.10.107.188" port: 34039 } }
20:07:02.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:02.726560 10927 sys_catalog.cc:455] T 00000000000000000000000000000000 P ce27da99bbe3488ca876547f04030d92 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d51056216da443c0bd535a9d772c37c2" member_type: VOTER last_known_addr { host: "127.10.107.190" port: 42101 } } peers { permanent_uuid: "86124f48110c470db8b724135148aaf2" member_type: VOTER last_known_addr { host: "127.10.107.189" port: 36917 } } peers { permanent_uuid: "ce27da99bbe3488ca876547f04030d92" member_type: VOTER last_known_addr { host: "127.10.107.188" port: 34039 } } }
20:07:02.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:02.727533 10927 sys_catalog.cc:458] T 00000000000000000000000000000000 P ce27da99bbe3488ca876547f04030d92 [sys.catalog]: This master's current role is: FOLLOWER
20:07:02.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:02.728543 10907 sys_catalog.cc:564] T 00000000000000000000000000000000 P ce27da99bbe3488ca876547f04030d92 [sys.catalog]: configured and running, proceeding with master startup.
20:07:02.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:02.754129 10936 catalog_manager.cc:1560] T 00000000000000000000000000000000 P 86124f48110c470db8b724135148aaf2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:07:02.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:02.755278 10936 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:07:02.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:02.742136 10752 master_runner.cc:186] Error getting master registration for 127.10.107.190:42101: OK, instance_id { permanent_uuid: "d51056216da443c0bd535a9d772c37c2" instance_seqno: 1757102818438075 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
20:07:02.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:02.783514 10752 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:07:02.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:02.789773 10861 tablet.cc:2378] T 00000000000000000000000000000000 P ce27da99bbe3488ca876547f04030d92: Can't schedule compaction. Clean time has not been advanced past its initial value.
20:07:02.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:02.792425 10944 catalog_manager.cc:1560] T 00000000000000000000000000000000 P ce27da99bbe3488ca876547f04030d92: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:07:02.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:02.793395 10944 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:07:02.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:02.807451 10919 raft_consensus.cc:491] T 00000000000000000000000000000000 P 86124f48110c470db8b724135148aaf2 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:07:02.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:02.808398 10919 raft_consensus.cc:513] T 00000000000000000000000000000000 P 86124f48110c470db8b724135148aaf2 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d51056216da443c0bd535a9d772c37c2" member_type: VOTER last_known_addr { host: "127.10.107.190" port: 42101 } } peers { permanent_uuid: "86124f48110c470db8b724135148aaf2" member_type: VOTER last_known_addr { host: "127.10.107.189" port: 36917 } } peers { permanent_uuid: "ce27da99bbe3488ca876547f04030d92" member_type: VOTER last_known_addr { host: "127.10.107.188" port: 34039 } }
20:07:02.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:02.812582 10882 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "86124f48110c470db8b724135148aaf2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ce27da99bbe3488ca876547f04030d92" is_pre_election: true
20:07:02.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:02.813951 10882 raft_consensus.cc:2466] T 00000000000000000000000000000000 P ce27da99bbe3488ca876547f04030d92 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 86124f48110c470db8b724135148aaf2 in term 0.
20:07:02.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:02.815151 10777 leader_election.cc:304] T 00000000000000000000000000000000 P 86124f48110c470db8b724135148aaf2 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 86124f48110c470db8b724135148aaf2, ce27da99bbe3488ca876547f04030d92; no voters:
20:07:02.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:02.783022 10827 master_runner.cc:186] Error getting master registration for 127.10.107.190:42101: OK, instance_id { permanent_uuid: "d51056216da443c0bd535a9d772c37c2" instance_seqno: 1757102818438075 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
20:07:02.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:02.818600 10827 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:07:02.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:02.819700 10946 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 86124f48110c470db8b724135148aaf2 [term 0 FOLLOWER]: Leader pre-election won for term 1
20:07:02.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:02.820358 10946 raft_consensus.cc:491] T 00000000000000000000000000000000 P 86124f48110c470db8b724135148aaf2 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
20:07:02.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:02.820914 10946 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 86124f48110c470db8b724135148aaf2 [term 0 FOLLOWER]: Advancing to term 1
20:07:02.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:02.825924 10919 leader_election.cc:290] T 00000000000000000000000000000000 P 86124f48110c470db8b724135148aaf2 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers d51056216da443c0bd535a9d772c37c2 (127.10.107.190:42101), ce27da99bbe3488ca876547f04030d92 (127.10.107.188:34039)
20:07:02.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:02.830832 10946 raft_consensus.cc:513] T 00000000000000000000000000000000 P 86124f48110c470db8b724135148aaf2 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d51056216da443c0bd535a9d772c37c2" member_type: VOTER last_known_addr { host: "127.10.107.190" port: 42101 } } peers { permanent_uuid: "86124f48110c470db8b724135148aaf2" member_type: VOTER last_known_addr { host: "127.10.107.189" port: 36917 } } peers { permanent_uuid: "ce27da99bbe3488ca876547f04030d92" member_type: VOTER last_known_addr { host: "127.10.107.188" port: 34039 } }
20:07:02.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:02.842041 10946 leader_election.cc:290] T 00000000000000000000000000000000 P 86124f48110c470db8b724135148aaf2 [CANDIDATE]: Term 1 election: Requested vote from peers d51056216da443c0bd535a9d772c37c2 (127.10.107.190:42101), ce27da99bbe3488ca876547f04030d92 (127.10.107.188:34039)
20:07:02.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:02.842985 10882 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "86124f48110c470db8b724135148aaf2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ce27da99bbe3488ca876547f04030d92"
20:07:02.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:02.843535 10882 raft_consensus.cc:3058] T 00000000000000000000000000000000 P ce27da99bbe3488ca876547f04030d92 [term 0 FOLLOWER]: Advancing to term 1
20:07:02.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:02.851136 10882 raft_consensus.cc:2466] T 00000000000000000000000000000000 P ce27da99bbe3488ca876547f04030d92 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 86124f48110c470db8b724135148aaf2 in term 1.
20:07:02.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:02.852401 10777 leader_election.cc:304] T 00000000000000000000000000000000 P 86124f48110c470db8b724135148aaf2 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 86124f48110c470db8b724135148aaf2, ce27da99bbe3488ca876547f04030d92; no voters:
20:07:02.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:02.853250 10946 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 86124f48110c470db8b724135148aaf2 [term 1 FOLLOWER]: Leader election won for term 1
20:07:02.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:02.853734 10946 raft_consensus.cc:695] T 00000000000000000000000000000000 P 86124f48110c470db8b724135148aaf2 [term 1 LEADER]: Becoming Leader. State: Replica: 86124f48110c470db8b724135148aaf2, State: Running, Role: LEADER
20:07:02.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:02.859230 10946 consensus_queue.cc:237] T 00000000000000000000000000000000 P 86124f48110c470db8b724135148aaf2 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d51056216da443c0bd535a9d772c37c2" member_type: VOTER last_known_addr { host: "127.10.107.190" port: 42101 } } peers { permanent_uuid: "86124f48110c470db8b724135148aaf2" member_type: VOTER last_known_addr { host: "127.10.107.189" port: 36917 } } peers { permanent_uuid: "ce27da99bbe3488ca876547f04030d92" member_type: VOTER last_known_addr { host: "127.10.107.188" port: 34039 } }
20:07:02.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:02.868427 10919 sys_catalog.cc:455] T 00000000000000000000000000000000 P 86124f48110c470db8b724135148aaf2 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 86124f48110c470db8b724135148aaf2. Latest consensus state: current_term: 1 leader_uuid: "86124f48110c470db8b724135148aaf2" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d51056216da443c0bd535a9d772c37c2" member_type: VOTER last_known_addr { host: "127.10.107.190" port: 42101 } } peers { permanent_uuid: "86124f48110c470db8b724135148aaf2" member_type: VOTER last_known_addr { host: "127.10.107.189" port: 36917 } } peers { permanent_uuid: "ce27da99bbe3488ca876547f04030d92" member_type: VOTER last_known_addr { host: "127.10.107.188" port: 34039 } } }
20:07:02.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:02.868885 10919 sys_catalog.cc:458] T 00000000000000000000000000000000 P 86124f48110c470db8b724135148aaf2 [sys.catalog]: This master's current role is: LEADER
20:07:02.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:02.874291 10948 catalog_manager.cc:1477] Loading table and tablet metadata into memory...
20:07:02.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:02.881791 10948 catalog_manager.cc:1486] Initializing Kudu cluster ID...
20:07:02.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:02.908327 10882 raft_consensus.cc:1273] T 00000000000000000000000000000000 P ce27da99bbe3488ca876547f04030d92 [term 1 FOLLOWER]: Refusing update from remote peer 86124f48110c470db8b724135148aaf2: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
20:07:02.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:02.910636 10919 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 86124f48110c470db8b724135148aaf2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce27da99bbe3488ca876547f04030d92" member_type: VOTER last_known_addr { host: "127.10.107.188" port: 34039 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
20:07:02.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:02.915695 10779 tablet.cc:2378] T 00000000000000000000000000000000 P 86124f48110c470db8b724135148aaf2: Can't schedule compaction. Clean time has not been advanced past its initial value.
20:07:02.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:02.939913 10927 sys_catalog.cc:455] T 00000000000000000000000000000000 P ce27da99bbe3488ca876547f04030d92 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 86124f48110c470db8b724135148aaf2. Latest consensus state: current_term: 1 leader_uuid: "86124f48110c470db8b724135148aaf2" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d51056216da443c0bd535a9d772c37c2" member_type: VOTER last_known_addr { host: "127.10.107.190" port: 42101 } } peers { permanent_uuid: "86124f48110c470db8b724135148aaf2" member_type: VOTER last_known_addr { host: "127.10.107.189" port: 36917 } } peers { permanent_uuid: "ce27da99bbe3488ca876547f04030d92" member_type: VOTER last_known_addr { host: "127.10.107.188" port: 34039 } } }
20:07:02.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:02.940642 10927 sys_catalog.cc:458] T 00000000000000000000000000000000 P ce27da99bbe3488ca876547f04030d92 [sys.catalog]: This master's current role is: FOLLOWER
20:07:02.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:02.966817 10927 sys_catalog.cc:455] T 00000000000000000000000000000000 P ce27da99bbe3488ca876547f04030d92 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "86124f48110c470db8b724135148aaf2" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d51056216da443c0bd535a9d772c37c2" member_type: VOTER last_known_addr { host: "127.10.107.190" port: 42101 } } peers { permanent_uuid: "86124f48110c470db8b724135148aaf2" member_type: VOTER last_known_addr { host: "127.10.107.189" port: 36917 } } peers { permanent_uuid: "ce27da99bbe3488ca876547f04030d92" member_type: VOTER last_known_addr { host: "127.10.107.188" port: 34039 } } }
20:07:02.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:02.957398 10950 mvcc.cc:204] Tried to move back new op lower bound from 7197093162617905152 to 7197093162441347072. Current Snapshot: MvccSnapshot[applied={T|T < 7197093162617905152}]
20:07:02.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:02.967628 10927 sys_catalog.cc:458] T 00000000000000000000000000000000 P ce27da99bbe3488ca876547f04030d92 [sys.catalog]: This master's current role is: FOLLOWER
20:07:02.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:02.981823 10948 catalog_manager.cc:1349] Generated new cluster ID: f955dc0c82cd496c9e937987724e4dbe
20:07:02.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:02.982136 10948 catalog_manager.cc:1497] Initializing Kudu internal certificate authority...
20:07:02.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:02.990263 10919 sys_catalog.cc:455] T 00000000000000000000000000000000 P 86124f48110c470db8b724135148aaf2 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "86124f48110c470db8b724135148aaf2" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d51056216da443c0bd535a9d772c37c2" member_type: VOTER last_known_addr { host: "127.10.107.190" port: 42101 } } peers { permanent_uuid: "86124f48110c470db8b724135148aaf2" member_type: VOTER last_known_addr { host: "127.10.107.189" port: 36917 } } peers { permanent_uuid: "ce27da99bbe3488ca876547f04030d92" member_type: VOTER last_known_addr { host: "127.10.107.188" port: 34039 } } }
20:07:02.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:02.990947 10919 sys_catalog.cc:458] T 00000000000000000000000000000000 P 86124f48110c470db8b724135148aaf2 [sys.catalog]: This master's current role is: LEADER
20:07:03.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:03.019539 10948 catalog_manager.cc:1372] Generated new certificate authority record
20:07:03.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:03.024222 10948 catalog_manager.cc:1506] Loading token signing keys...
20:07:03.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:03.049278 10948 catalog_manager.cc:5955] T 00000000000000000000000000000000 P 86124f48110c470db8b724135148aaf2: Generated new TSK 0
20:07:03.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:03.050717 10948 catalog_manager.cc:1516] Initializing in-progress tserver states...
20:07:03.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:03.065847 10670 external_mini_cluster.cc:1366] Running /tmp/dist-test-taske7evVR/build/tsan/bin/kudu
20:07:03.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taske7evVR/build/tsan/bin/kudu
20:07:03.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster2568307083494362251/ts-0/wal
20:07:03.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster2568307083494362251/ts-0/data
20:07:03.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:07:03.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:07:03.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:07:03.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:07:03.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:07:03.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:07:03.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster2568307083494362251/ts-0/logs
20:07:03.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster2568307083494362251/ts-0/data/info.pb
20:07:03.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:07:03.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:07:03.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:07:03.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:07:03.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:07:03.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:07:03.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:07:03.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:07:03.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
20:07:03.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:07:03.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.107.129:0
20:07:03.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.10.107.129
20:07:03.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.107.129
20:07:03.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:07:03.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.10.107.190:42101,127.10.107.189:36917,127.10.107.188:34039
20:07:03.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.107.148:41887
20:07:03.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:07:03.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:07:03.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
20:07:03.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:03.196323 10750 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.107.188" port: 34039 } attempt: 7
20:07:03.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:03.262697 10750 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d51056216da443c0bd535a9d772c37c2: Bootstrap starting.
20:07:03.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:03.285910 10750 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P d51056216da443c0bd535a9d772c37c2: Neither blocks nor log segments found. Creating new log.
20:07:03.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:03.291733 10750 log.cc:826] T 00000000000000000000000000000000 P d51056216da443c0bd535a9d772c37c2: Log is configured to *not* fsync() on all Append() calls
20:07:03.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:03.304906 10750 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d51056216da443c0bd535a9d772c37c2: No bootstrap required, opened a new log
20:07:03.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:03.373390 10750 raft_consensus.cc:357] T 00000000000000000000000000000000 P d51056216da443c0bd535a9d772c37c2 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d51056216da443c0bd535a9d772c37c2" member_type: VOTER last_known_addr { host: "127.10.107.190" port: 42101 } } peers { permanent_uuid: "86124f48110c470db8b724135148aaf2" member_type: VOTER last_known_addr { host: "127.10.107.189" port: 36917 } } peers { permanent_uuid: "ce27da99bbe3488ca876547f04030d92" member_type: VOTER last_known_addr { host: "127.10.107.188" port: 34039 } }
20:07:03.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:03.375581 10750 raft_consensus.cc:383] T 00000000000000000000000000000000 P d51056216da443c0bd535a9d772c37c2 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:07:03.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:03.376410 10750 raft_consensus.cc:738] T 00000000000000000000000000000000 P d51056216da443c0bd535a9d772c37c2 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d51056216da443c0bd535a9d772c37c2, State: Initialized, Role: FOLLOWER
20:07:03.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:03.379159 10750 consensus_queue.cc:260] T 00000000000000000000000000000000 P d51056216da443c0bd535a9d772c37c2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d51056216da443c0bd535a9d772c37c2" member_type: VOTER last_known_addr { host: "127.10.107.190" port: 42101 } } peers { permanent_uuid: "86124f48110c470db8b724135148aaf2" member_type: VOTER last_known_addr { host: "127.10.107.189" port: 36917 } } peers { permanent_uuid: "ce27da99bbe3488ca876547f04030d92" member_type: VOTER last_known_addr { host: "127.10.107.188" port: 34039 } }
20:07:03.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:03.403064 10724 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "86124f48110c470db8b724135148aaf2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d51056216da443c0bd535a9d772c37c2"
20:07:03.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:03.403795 10724 raft_consensus.cc:3058] T 00000000000000000000000000000000 P d51056216da443c0bd535a9d772c37c2 [term 0 FOLLOWER]: Advancing to term 1
20:07:03.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:03.405810 10725 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "86124f48110c470db8b724135148aaf2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d51056216da443c0bd535a9d772c37c2" is_pre_election: true
20:07:03.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:03.415892 10724 raft_consensus.cc:2466] T 00000000000000000000000000000000 P d51056216da443c0bd535a9d772c37c2 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 86124f48110c470db8b724135148aaf2 in term 1.
20:07:03.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:03.418130 10723 raft_consensus.cc:1273] T 00000000000000000000000000000000 P d51056216da443c0bd535a9d772c37c2 [term 1 FOLLOWER]: Refusing update from remote peer 86124f48110c470db8b724135148aaf2: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
20:07:03.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:03.422561 10919 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 86124f48110c470db8b724135148aaf2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d51056216da443c0bd535a9d772c37c2" member_type: VOTER last_known_addr { host: "127.10.107.190" port: 42101 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
20:07:03.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:03.416656 10750 sys_catalog.cc:564] T 00000000000000000000000000000000 P d51056216da443c0bd535a9d772c37c2 [sys.catalog]: configured and running, proceeding with master startup.
20:07:03.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:03.401348 10958 sys_catalog.cc:455] T 00000000000000000000000000000000 P d51056216da443c0bd535a9d772c37c2 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d51056216da443c0bd535a9d772c37c2" member_type: VOTER last_known_addr { host: "127.10.107.190" port: 42101 } } peers { permanent_uuid: "86124f48110c470db8b724135148aaf2" member_type: VOTER last_known_addr { host: "127.10.107.189" port: 36917 } } peers { permanent_uuid: "ce27da99bbe3488ca876547f04030d92" member_type: VOTER last_known_addr { host: "127.10.107.188" port: 34039 } } }
20:07:03.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:03.432719 10958 sys_catalog.cc:458] T 00000000000000000000000000000000 P d51056216da443c0bd535a9d772c37c2 [sys.catalog]: This master's current role is: FOLLOWER
20:07:03.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:03.499186 10958 sys_catalog.cc:455] T 00000000000000000000000000000000 P d51056216da443c0bd535a9d772c37c2 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 86124f48110c470db8b724135148aaf2. Latest consensus state: current_term: 1 leader_uuid: "86124f48110c470db8b724135148aaf2" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d51056216da443c0bd535a9d772c37c2" member_type: VOTER last_known_addr { host: "127.10.107.190" port: 42101 } } peers { permanent_uuid: "86124f48110c470db8b724135148aaf2" member_type: VOTER last_known_addr { host: "127.10.107.189" port: 36917 } } peers { permanent_uuid: "ce27da99bbe3488ca876547f04030d92" member_type: VOTER last_known_addr { host: "127.10.107.188" port: 34039 } } }
20:07:03.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:03.499902 10958 sys_catalog.cc:458] T 00000000000000000000000000000000 P d51056216da443c0bd535a9d772c37c2 [sys.catalog]: This master's current role is: FOLLOWER
20:07:03.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:03.502640 10959 sys_catalog.cc:455] T 00000000000000000000000000000000 P d51056216da443c0bd535a9d772c37c2 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "86124f48110c470db8b724135148aaf2" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d51056216da443c0bd535a9d772c37c2" member_type: VOTER last_known_addr { host: "127.10.107.190" port: 42101 } } peers { permanent_uuid: "86124f48110c470db8b724135148aaf2" member_type: VOTER last_known_addr { host: "127.10.107.189" port: 36917 } } peers { permanent_uuid: "ce27da99bbe3488ca876547f04030d92" member_type: VOTER last_known_addr { host: "127.10.107.188" port: 34039 } } }
20:07:03.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:03.503399 10959 sys_catalog.cc:458] T 00000000000000000000000000000000 P d51056216da443c0bd535a9d772c37c2 [sys.catalog]: This master's current role is: FOLLOWER
20:07:03.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:03.582315 10978 catalog_manager.cc:1261] Loaded cluster ID: f955dc0c82cd496c9e937987724e4dbe
20:07:03.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:03.588090 10978 catalog_manager.cc:1554] T 00000000000000000000000000000000 P d51056216da443c0bd535a9d772c37c2: loading cluster ID for follower catalog manager: success
20:07:03.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:03.601401 10954 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:07:03.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:03.602356 10954 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:07:03.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:03.603463 10954 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:07:03.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:03.613917 10978 catalog_manager.cc:1576] T 00000000000000000000000000000000 P d51056216da443c0bd535a9d772c37c2: acquiring CA information for follower catalog manager: success
20:07:03.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:03.624536 10978 catalog_manager.cc:1604] T 00000000000000000000000000000000 P d51056216da443c0bd535a9d772c37c2: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
20:07:03.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:03.660604 10954 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:07:03.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:03.662441 10954 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.10.107.129
20:07:03.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:03.724237 10954 tablet_server_runner.cc:78] Tablet server non-default flags:
20:07:03.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:07:03.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.107.148:41887
20:07:03.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:07:03.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:07:03.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster2568307083494362251/ts-0/data
20:07:03.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster2568307083494362251/ts-0/wal
20:07:03.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:07:03.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:07:03.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.107.129:0
20:07:03.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:07:03.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:07:03.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:07:03.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster2568307083494362251/ts-0/data/info.pb
20:07:03.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.107.129
20:07:03.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:07:03.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.10.107.190:42101,127.10.107.189:36917,127.10.107.188:34039
20:07:03.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:07:03.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:07:03.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:07:03.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:07:03.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:07:03.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.10.107.129
20:07:03.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster2568307083494362251/ts-0/logs
20:07:03.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:07:03.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:07:03.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:07:03.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
20:07:03.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:07:03.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision e2e116a9cb215058aad161061214d11c617cedac
20:07:03.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:07:03.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 05 Sep 2025 19:43:22 UTC on 5fd53c4cbb9d
20:07:03.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7917
20:07:03.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:07:03.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:03.733139 10954 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:07:03.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:03.735833 10954 file_cache.cc:492] Constructed file cache file cache with capacity 419430
20:07:03.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:03.787910 10981 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:07:03.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:03.799052 10944 catalog_manager.cc:1261] Loaded cluster ID: f955dc0c82cd496c9e937987724e4dbe
20:07:03.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:03.799741 10944 catalog_manager.cc:1554] T 00000000000000000000000000000000 P ce27da99bbe3488ca876547f04030d92: loading cluster ID for follower catalog manager: success
20:07:03.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:03.807778 10944 catalog_manager.cc:1576] T 00000000000000000000000000000000 P ce27da99bbe3488ca876547f04030d92: acquiring CA information for follower catalog manager: success
20:07:03.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:03.817430 10944 catalog_manager.cc:1604] T 00000000000000000000000000000000 P ce27da99bbe3488ca876547f04030d92: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
20:07:03.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:03.912010 10985 sys_catalog.cc:455] T 00000000000000000000000000000000 P 86124f48110c470db8b724135148aaf2 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "86124f48110c470db8b724135148aaf2" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d51056216da443c0bd535a9d772c37c2" member_type: VOTER last_known_addr { host: "127.10.107.190" port: 42101 } } peers { permanent_uuid: "86124f48110c470db8b724135148aaf2" member_type: VOTER last_known_addr { host: "127.10.107.189" port: 36917 } } peers { permanent_uuid: "ce27da99bbe3488ca876547f04030d92" member_type: VOTER last_known_addr { host: "127.10.107.188" port: 34039 } } }
20:07:03.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:03.920269 10985 sys_catalog.cc:458] T 00000000000000000000000000000000 P 86124f48110c470db8b724135148aaf2 [sys.catalog]: This master's current role is: LEADER
20:07:05.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:03.813278 10982 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:07:05.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:05.590390 10954 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 1.798s user 0.360s sys 0.578s
20:07:05.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:05.591068 10954 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 1.798s user 0.361s sys 0.579s
20:07:05.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:05.158147 10980 debug-util.cc:398] Leaking SignalData structure 0x7b0800034ea0 after lost signal to thread 10954
20:07:05.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:05.618005 10954 server_base.cc:1047] running on GCE node
20:07:05.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:05.619645 10988 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:07:05.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:05.622480 10954 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:07:05.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:05.637970 10954 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:07:05.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:05.639786 10954 hybrid_clock.cc:648] HybridClock initialized: now 1757102825639745 us; error 47 us; skew 500 ppm
20:07:05.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:05.640921 10954 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:07:05.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:05.671787 10954 webserver.cc:480] Webserver started at http://127.10.107.129:38875/ using document root <none> and password file <none>
20:07:05.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:05.673530 10954 fs_manager.cc:362] Metadata directory not provided
20:07:05.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:05.673921 10954 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:07:05.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:05.674603 10954 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
20:07:05.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:05.681068 10954 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster2568307083494362251/ts-0/data/instance:
20:07:05.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c5c2437fd41f43de9620ca1fcf1f81cf"
20:07:05.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-05 20:07:05 on dist-test-slave-x5g2"
20:07:05.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:05.682813 10954 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster2568307083494362251/ts-0/wal/instance:
20:07:05.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c5c2437fd41f43de9620ca1fcf1f81cf"
20:07:05.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-05 20:07:05 on dist-test-slave-x5g2"
20:07:05.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:05.698139 10954 fs_manager.cc:696] Time spent creating directory manager: real 0.014s user 0.004s sys 0.009s
20:07:05.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:05.712505 10994 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:07:05.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:05.718884 10954 fs_manager.cc:730] Time spent opening block manager: real 0.016s user 0.006s sys 0.002s
20:07:05.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:05.720194 10954 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster2568307083494362251/ts-0/data,/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster2568307083494362251/ts-0/wal
20:07:05.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c5c2437fd41f43de9620ca1fcf1f81cf"
20:07:05.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-05 20:07:05 on dist-test-slave-x5g2"
20:07:05.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:05.721027 10954 fs_report.cc:389] FS layout report
20:07:05.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:07:05.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster2568307083494362251/ts-0/wal
20:07:05.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster2568307083494362251/ts-0/wal
20:07:05.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster2568307083494362251/ts-0/data/data
20:07:05.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:07:05.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:07:05.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:07:05.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:07:05.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:07:05.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:07:05.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:07:05.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:07:05.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:07:05.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:07:05.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:07:05.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:05.814702 10954 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:07:05.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:05.816890 10954 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:07:05.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:05.817670 10954 kserver.cc:163] Server-wide thread pool size limit: 3276
20:07:05.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:05.827813 10954 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
20:07:05.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:05.840135 10954 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
20:07:05.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:05.840703 10954 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.001s user 0.000s sys 0.000s
20:07:05.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:05.841200 10954 ts_tablet_manager.cc:610] Registered 0 tablets
20:07:05.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:05.841580 10954 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
20:07:06.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:06.200762 10954 rpc_server.cc:307] RPC server started. Bound to: 127.10.107.129:36553
20:07:06.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:06.200888 11108 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.10.107.129:36553 every 8 connection(s)
20:07:06.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:06.203896 10954 server_base.cc:1179] Dumped server information to /tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster2568307083494362251/ts-0/data/info.pb
20:07:06.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:06.207571 10670 external_mini_cluster.cc:1428] Started /tmp/dist-test-taske7evVR/build/tsan/bin/kudu as pid 10954
20:07:06.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:06.208325 10670 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster2568307083494362251/ts-0/wal/instance
20:07:06.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:06.219872 10670 external_mini_cluster.cc:1366] Running /tmp/dist-test-taske7evVR/build/tsan/bin/kudu
20:07:06.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taske7evVR/build/tsan/bin/kudu
20:07:06.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster2568307083494362251/ts-1/wal
20:07:06.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster2568307083494362251/ts-1/data
20:07:06.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:07:06.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:07:06.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:07:06.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:07:06.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:07:06.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:07:06.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster2568307083494362251/ts-1/logs
20:07:06.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster2568307083494362251/ts-1/data/info.pb
20:07:06.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:07:06.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:07:06.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:07:06.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:07:06.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:07:06.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:07:06.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:07:06.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:07:06.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
20:07:06.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:07:06.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.107.130:0
20:07:06.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.10.107.130
20:07:06.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.107.130
20:07:06.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:07:06.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.10.107.190:42101,127.10.107.189:36917,127.10.107.188:34039
20:07:06.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.107.148:41887
20:07:06.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:07:06.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:07:06.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
20:07:06.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:06.315405 11109 heartbeater.cc:344] Connected to a master server at 127.10.107.188:34039
20:07:06.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:06.316462 11109 heartbeater.cc:461] Registering TS with master...
20:07:06.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:06.317797 11109 heartbeater.cc:507] Master 127.10.107.188:34039 requested a full tablet report, sending...
20:07:06.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:06.321157 11110 heartbeater.cc:344] Connected to a master server at 127.10.107.189:36917
20:07:06.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:06.321506 11110 heartbeater.cc:461] Registering TS with master...
20:07:06.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:06.322362 11110 heartbeater.cc:507] Master 127.10.107.189:36917 requested a full tablet report, sending...
20:07:06.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:06.322902 10872 ts_manager.cc:194] Registered new tserver with Master: c5c2437fd41f43de9620ca1fcf1f81cf (127.10.107.129:36553)
20:07:06.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:06.325362 10790 ts_manager.cc:194] Registered new tserver with Master: c5c2437fd41f43de9620ca1fcf1f81cf (127.10.107.129:36553)
20:07:06.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:06.327880 10790 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.10.107.129:36177
20:07:06.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:06.336380 11111 heartbeater.cc:344] Connected to a master server at 127.10.107.190:42101
20:07:06.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:06.336736 11111 heartbeater.cc:461] Registering TS with master...
20:07:06.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:06.337332 11111 heartbeater.cc:507] Master 127.10.107.190:42101 requested a full tablet report, sending...
20:07:06.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:06.340791 10715 ts_manager.cc:194] Registered new tserver with Master: c5c2437fd41f43de9620ca1fcf1f81cf (127.10.107.129:36553)
20:07:06.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:06.706646 11116 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:07:06.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:06.707171 11116 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:07:06.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:06.707794 11116 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:07:06.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:06.740144 11116 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:07:06.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:06.741132 11116 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.10.107.130
20:07:06.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:06.775717 11116 tablet_server_runner.cc:78] Tablet server non-default flags:
20:07:06.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:07:06.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.107.148:41887
20:07:06.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:07:06.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:07:06.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster2568307083494362251/ts-1/data
20:07:06.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster2568307083494362251/ts-1/wal
20:07:06.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:07:06.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:07:06.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.107.130:0
20:07:06.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:07:06.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:07:06.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:07:06.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster2568307083494362251/ts-1/data/info.pb
20:07:06.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.107.130
20:07:06.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:07:06.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.10.107.190:42101,127.10.107.189:36917,127.10.107.188:34039
20:07:06.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:07:06.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:07:06.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:07:06.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:07:06.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:07:06.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.10.107.130
20:07:06.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster2568307083494362251/ts-1/logs
20:07:06.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:07:06.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:07:06.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:07:06.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
20:07:06.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:07:06.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision e2e116a9cb215058aad161061214d11c617cedac
20:07:06.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:07:06.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 05 Sep 2025 19:43:22 UTC on 5fd53c4cbb9d
20:07:06.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7917
20:07:06.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:07:06.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:06.777381 11116 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:07:06.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:06.779146 11116 file_cache.cc:492] Constructed file cache file cache with capacity 419430
20:07:06.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:06.792282 11126 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:07:07.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:07.331990 11110 heartbeater.cc:499] Master 127.10.107.189:36917 was elected leader, sending a full tablet report...
20:07:08.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:06.794253 11127 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:07:08.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:08.029038 11129 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:07:08.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:08.032534 11128 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1238 milliseconds
20:07:08.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:08.032625 11116 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:07:08.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:08.033818 11116 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:07:08.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:08.036017 11116 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:07:08.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:08.037369 11116 hybrid_clock.cc:648] HybridClock initialized: now 1757102828037317 us; error 49 us; skew 500 ppm
20:07:08.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:08.038259 11116 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:07:08.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:08.045150 11116 webserver.cc:480] Webserver started at http://127.10.107.130:42119/ using document root <none> and password file <none>
20:07:08.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:08.046170 11116 fs_manager.cc:362] Metadata directory not provided
20:07:08.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:08.046396 11116 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:07:08.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:08.046952 11116 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
20:07:08.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:08.051335 11116 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster2568307083494362251/ts-1/data/instance:
20:07:08.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d425196718034c149f798db6655d8bfd"
20:07:08.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-05 20:07:08 on dist-test-slave-x5g2"
20:07:08.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:08.052368 11116 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster2568307083494362251/ts-1/wal/instance:
20:07:08.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d425196718034c149f798db6655d8bfd"
20:07:08.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-05 20:07:08 on dist-test-slave-x5g2"
20:07:08.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:08.059724 11116 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.005s sys 0.004s
20:07:08.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:08.066440 11136 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:07:08.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:08.067452 11116 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.004s sys 0.000s
20:07:08.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:08.067730 11116 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster2568307083494362251/ts-1/data,/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster2568307083494362251/ts-1/wal
20:07:08.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d425196718034c149f798db6655d8bfd"
20:07:08.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-05 20:07:08 on dist-test-slave-x5g2"
20:07:08.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:08.068018 11116 fs_report.cc:389] FS layout report
20:07:08.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:07:08.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster2568307083494362251/ts-1/wal
20:07:08.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster2568307083494362251/ts-1/wal
20:07:08.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster2568307083494362251/ts-1/data/data
20:07:08.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:07:08.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:07:08.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:07:08.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:07:08.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:07:08.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:07:08.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:07:08.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:07:08.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:07:08.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:07:08.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:07:08.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:08.122714 11116 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:07:08.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:08.124677 11116 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:07:08.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:08.125072 11116 kserver.cc:163] Server-wide thread pool size limit: 3276
20:07:08.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:08.127491 11116 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
20:07:08.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:08.131912 11116 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
20:07:08.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:08.132118 11116 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
20:07:08.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:08.132304 11116 ts_tablet_manager.cc:610] Registered 0 tablets
20:07:08.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:08.132436 11116 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
20:07:08.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:08.294062 11116 rpc_server.cc:307] RPC server started. Bound to: 127.10.107.130:34831
20:07:08.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:08.294165 11249 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.10.107.130:34831 every 8 connection(s)
20:07:08.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:08.296509 11116 server_base.cc:1179] Dumped server information to /tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster2568307083494362251/ts-1/data/info.pb
20:07:08.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:08.301618 10670 external_mini_cluster.cc:1428] Started /tmp/dist-test-taske7evVR/build/tsan/bin/kudu as pid 11116
20:07:08.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:08.302078 10670 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster2568307083494362251/ts-1/wal/instance
20:07:08.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:08.324015 10670 external_mini_cluster.cc:1366] Running /tmp/dist-test-taske7evVR/build/tsan/bin/kudu
20:07:08.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taske7evVR/build/tsan/bin/kudu
20:07:08.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster2568307083494362251/ts-2/wal
20:07:08.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster2568307083494362251/ts-2/data
20:07:08.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:07:08.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:07:08.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:07:08.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:07:08.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:07:08.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:07:08.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster2568307083494362251/ts-2/logs
20:07:08.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster2568307083494362251/ts-2/data/info.pb
20:07:08.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:07:08.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:07:08.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:07:08.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:07:08.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:07:08.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:07:08.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:07:08.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:07:08.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
20:07:08.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:07:08.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.107.131:0
20:07:08.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.10.107.131
20:07:08.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.107.131
20:07:08.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:07:08.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.10.107.190:42101,127.10.107.189:36917,127.10.107.188:34039
20:07:08.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.107.148:41887
20:07:08.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:07:08.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:07:08.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
20:07:08.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:08.342551 11252 heartbeater.cc:344] Connected to a master server at 127.10.107.190:42101
20:07:08.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:08.343166 11252 heartbeater.cc:461] Registering TS with master...
20:07:08.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:08.344245 11250 heartbeater.cc:344] Connected to a master server at 127.10.107.188:34039
20:07:08.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:08.344472 11252 heartbeater.cc:507] Master 127.10.107.190:42101 requested a full tablet report, sending...
20:07:08.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:08.344615 11250 heartbeater.cc:461] Registering TS with master...
20:07:08.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:08.346235 11251 heartbeater.cc:344] Connected to a master server at 127.10.107.189:36917
20:07:08.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:08.346575 11251 heartbeater.cc:461] Registering TS with master...
20:07:08.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:08.347162 11250 heartbeater.cc:507] Master 127.10.107.188:34039 requested a full tablet report, sending...
20:07:08.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:08.347364 11251 heartbeater.cc:507] Master 127.10.107.189:36917 requested a full tablet report, sending...
20:07:08.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:08.348249 10715 ts_manager.cc:194] Registered new tserver with Master: d425196718034c149f798db6655d8bfd (127.10.107.130:34831)
20:07:08.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:08.350106 10790 ts_manager.cc:194] Registered new tserver with Master: d425196718034c149f798db6655d8bfd (127.10.107.130:34831)
20:07:08.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:08.350106 10872 ts_manager.cc:194] Registered new tserver with Master: d425196718034c149f798db6655d8bfd (127.10.107.130:34831)
20:07:08.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:08.351560 10790 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.10.107.130:39285
20:07:08.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:08.657732 11261 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:07:08.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:08.658162 11261 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:07:08.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:08.658603 11261 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:07:08.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:08.689486 11261 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:07:08.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:08.690217 11261 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.10.107.131
20:07:08.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:08.723557 11261 tablet_server_runner.cc:78] Tablet server non-default flags:
20:07:08.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:07:08.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.107.148:41887
20:07:08.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:07:08.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:07:08.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster2568307083494362251/ts-2/data
20:07:08.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster2568307083494362251/ts-2/wal
20:07:08.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:07:08.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:07:08.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.107.131:0
20:07:08.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:07:08.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:07:08.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:07:08.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster2568307083494362251/ts-2/data/info.pb
20:07:08.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.107.131
20:07:08.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:07:08.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.10.107.190:42101,127.10.107.189:36917,127.10.107.188:34039
20:07:08.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:07:08.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:07:08.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:07:08.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:07:08.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:07:08.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.10.107.131
20:07:08.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster2568307083494362251/ts-2/logs
20:07:08.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:07:08.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:07:08.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:07:08.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
20:07:08.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:07:08.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision e2e116a9cb215058aad161061214d11c617cedac
20:07:08.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:07:08.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 05 Sep 2025 19:43:22 UTC on 5fd53c4cbb9d
20:07:08.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7917
20:07:08.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:07:08.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:08.724957 11261 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:07:08.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:08.726544 11261 file_cache.cc:492] Constructed file cache file cache with capacity 419430
20:07:08.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:08.739881 11267 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:07:09.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:09.355295 11251 heartbeater.cc:499] Master 127.10.107.189:36917 was elected leader, sending a full tablet report...
20:07:09.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:08.741065 11268 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:07:09.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:08.744341 11270 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:07:09.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:09.946605 11269 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1200 milliseconds
20:07:09.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:09.946769 11261 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:07:09.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:09.948010 11261 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:07:09.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:09.950233 11261 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:07:09.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:09.951596 11261 hybrid_clock.cc:648] HybridClock initialized: now 1757102829951549 us; error 59 us; skew 500 ppm
20:07:09.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:09.952404 11261 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:07:09.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:09.958317 11261 webserver.cc:480] Webserver started at http://127.10.107.131:32961/ using document root <none> and password file <none>
20:07:09.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:09.959174 11261 fs_manager.cc:362] Metadata directory not provided
20:07:09.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:09.959352 11261 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:07:09.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:09.959769 11261 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
20:07:09.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:09.964258 11261 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster2568307083494362251/ts-2/data/instance:
20:07:09.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "76468a7c3e4b419c972934c32d3df884"
20:07:09.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-05 20:07:09 on dist-test-slave-x5g2"
20:07:09.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:09.965337 11261 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster2568307083494362251/ts-2/wal/instance:
20:07:09.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "76468a7c3e4b419c972934c32d3df884"
20:07:09.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-05 20:07:09 on dist-test-slave-x5g2"
20:07:09.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:09.972365 11261 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.005s sys 0.001s
20:07:09.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:09.978036 11278 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:07:09.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:09.979030 11261 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
20:07:09.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:09.979290 11261 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster2568307083494362251/ts-2/data,/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster2568307083494362251/ts-2/wal
20:07:09.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "76468a7c3e4b419c972934c32d3df884"
20:07:09.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-05 20:07:09 on dist-test-slave-x5g2"
20:07:09.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:09.979579 11261 fs_report.cc:389] FS layout report
20:07:09.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:07:09.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster2568307083494362251/ts-2/wal
20:07:09.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster2568307083494362251/ts-2/wal
20:07:09.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster2568307083494362251/ts-2/data/data
20:07:09.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:07:09.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:07:09.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:07:09.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:07:09.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:07:09.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:07:09.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:07:09.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:07:09.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:07:09.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:07:09.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:07:10.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:10.028870 11261 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:07:10.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:10.030272 11261 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:07:10.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:10.030656 11261 kserver.cc:163] Server-wide thread pool size limit: 3276
20:07:10.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:10.033056 11261 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
20:07:10.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:10.037127 11261 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
20:07:10.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:10.037317 11261 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
20:07:10.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:10.037528 11261 ts_tablet_manager.cc:610] Registered 0 tablets
20:07:10.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:10.037658 11261 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
20:07:10.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:10.185048 11261 rpc_server.cc:307] RPC server started. Bound to: 127.10.107.131:38751
20:07:10.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:10.185138 11390 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.10.107.131:38751 every 8 connection(s)
20:07:10.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:10.187635 11261 server_base.cc:1179] Dumped server information to /tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster2568307083494362251/ts-2/data/info.pb
20:07:10.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:10.194788 10670 external_mini_cluster.cc:1428] Started /tmp/dist-test-taske7evVR/build/tsan/bin/kudu as pid 11261
20:07:10.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:10.195251 10670 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster2568307083494362251/ts-2/wal/instance
20:07:10.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:10.238178 11393 heartbeater.cc:344] Connected to a master server at 127.10.107.190:42101
20:07:10.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:10.238678 11393 heartbeater.cc:461] Registering TS with master...
20:07:10.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:10.240599 11393 heartbeater.cc:507] Master 127.10.107.190:42101 requested a full tablet report, sending...
20:07:10.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:10.240803 11392 heartbeater.cc:344] Connected to a master server at 127.10.107.189:36917
20:07:10.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:10.241160 11392 heartbeater.cc:461] Registering TS with master...
20:07:10.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:10.242022 11392 heartbeater.cc:507] Master 127.10.107.189:36917 requested a full tablet report, sending...
20:07:10.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:10.243992 10714 ts_manager.cc:194] Registered new tserver with Master: 76468a7c3e4b419c972934c32d3df884 (127.10.107.131:38751)
20:07:10.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:10.244550 10790 ts_manager.cc:194] Registered new tserver with Master: 76468a7c3e4b419c972934c32d3df884 (127.10.107.131:38751)
20:07:10.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:10.245205 11391 heartbeater.cc:344] Connected to a master server at 127.10.107.188:34039
20:07:10.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:10.245536 11391 heartbeater.cc:461] Registering TS with master...
20:07:10.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:10.246351 11391 heartbeater.cc:507] Master 127.10.107.188:34039 requested a full tablet report, sending...
20:07:10.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:10.246577 10790 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.10.107.131:41325
20:07:10.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:10.248726 10872 ts_manager.cc:194] Registered new tserver with Master: 76468a7c3e4b419c972934c32d3df884 (127.10.107.131:38751)
20:07:10.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:10.264837 10670 external_mini_cluster.cc:949] 3 TS(s) registered with all masters
20:07:10.276 [DEBUG - main] (MiniKuduCluster.java:195) Response:
20:07:10.278 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
20:07:10.287 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.10.107.190"
port: 42101
}
bound_http_address {
host: "127.10.107.190"
port: 42735
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.10.107.189"
port: 36917
}
bound_http_address {
host: "127.10.107.189"
port: 44973
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.10.107.188"
port: 34039
}
bound_http_address {
host: "127.10.107.188"
port: 34309
}
}
}
20:07:10.291 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
20:07:10.294 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.10.107.129"
port: 36553
}
bound_http_address {
host: "127.10.107.129"
port: 38875
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.10.107.130"
port: 34831
}
bound_http_address {
host: "127.10.107.130"
port: 42119
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.10.107.131"
port: 38751
}
bound_http_address {
host: "127.10.107.131"
port: 32961
}
}
}
20:07:10.294 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
20:07:10.482 [DEBUG - main] (CompositeStrategy.java:59) Unable to load modern Subject methods, relying only on legacy methods
java.lang.NoSuchMethodException: javax.security.auth.Subject.current()
at java.lang.Class.getDeclaredMethod(Class.java:2130) ~[?:1.8.0_275]
at org.apache.kudu.client.internals.ModernStrategy.<init>(ModernStrategy.java:51) ~[main/:?]
at org.apache.kudu.client.internals.CompositeStrategy.<init>(CompositeStrategy.java:52) [main/:?]
at org.apache.kudu.client.internals.CompositeStrategy.<clinit>(CompositeStrategy.java:41) [main/:?]
at org.apache.kudu.client.internals.SecurityManagerCompatibility.get(SecurityManagerCompatibility.java:43) [main/:?]
at org.apache.kudu.client.SecurityContext.setupSubject(SecurityContext.java:155) [main/:?]
at org.apache.kudu.client.SecurityContext.<init>(SecurityContext.java:138) [main/:?]
at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:431) [main/:?]
at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:299) [main/:?]
at org.apache.kudu.client.AsyncKuduClient$AsyncKuduClientBuilder.build(AsyncKuduClient.java:3109) [main/:?]
at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:161) [kudu-test-utils-1.19.0-SNAPSHOT-unshaded.jar:1.19.0-SNAPSHOT]
at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50) [junit-4.13.2.jar:4.13.2]
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184) [kudu-test-utils-1.19.0-SNAPSHOT-unshaded.jar:1.19.0-SNAPSHOT]
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244) [kudu-test-utils-1.19.0-SNAPSHOT-unshaded.jar:1.19.0-SNAPSHOT]
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184) [kudu-test-utils-1.19.0-SNAPSHOT-unshaded.jar:1.19.0-SNAPSHOT]
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244) [kudu-test-utils-1.19.0-SNAPSHOT-unshaded.jar:1.19.0-SNAPSHOT]
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.run(ParentRunner.java:413) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.Suite.runChild(Suite.java:128) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.Suite.runChild(Suite.java:27) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.run(ParentRunner.java:413) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.run(JUnitCore.java:137) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.run(JUnitCore.java:115) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.main(JUnitCore.java:36) [junit-4.13.2.jar:4.13.2]
java.lang.NoSuchMethodException: javax.security.auth.Subject.current()
at java.lang.Class.getDeclaredMethod(Class.java:2130)
at org.apache.kudu.client.internals.ModernStrategy.<init>(ModernStrategy.java:51)
at org.apache.kudu.client.internals.CompositeStrategy.<init>(CompositeStrategy.java:52)
at org.apache.kudu.client.internals.CompositeStrategy.<clinit>(CompositeStrategy.java:41)
at org.apache.kudu.client.internals.SecurityManagerCompatibility.get(SecurityManagerCompatibility.java:43)
at org.apache.kudu.client.SecurityContext.setupSubject(SecurityContext.java:155)
at org.apache.kudu.client.SecurityContext.<init>(SecurityContext.java:138)
at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:431)
at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$AsyncKuduClientBuilder.build(AsyncKuduClient.java:3109)
at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:161)
at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runners.Suite.runChild(Suite.java:128)
at org.junit.runners.Suite.runChild(Suite.java:27)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runner.JUnitCore.run(JUnitCore.java:137)
at org.junit.runner.JUnitCore.run(JUnitCore.java:115)
at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77)
at org.junit.runner.JUnitCore.main(JUnitCore.java:36)
20:07:10.505 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
20:07:10.627 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taske7evVR/test-tmp
20:07:10.628 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
20:07:10.629 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taske7evVR/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
20:07:10.634 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: true
cluster_root: "/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073"
extra_master_flags: "--webserver_require_spnego=false"
extra_master_flags: "--authn_token_validity_seconds=10"
extra_tserver_flags: "--webserver_require_spnego=false"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
20:07:10.952 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:07:10.952 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
20:07:10.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:10.955195 11403 mini_oidc.cc:150] Starting JWKS server
20:07:10.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:10.958329 11403 webserver.cc:480] Webserver started at http://127.0.0.1:34155/ using document root <none> and password file <none>
20:07:10.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:10.958743 11403 mini_oidc.cc:186] Starting OIDC Discovery server
20:07:10.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:10.960840 11403 webserver.cc:480] Webserver started at http://127.0.0.1:39033/ using document root <none> and password file <none>
20:07:11.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
20:07:11.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/krb5kdc/principal' for realm 'KRBTEST.COM',
20:07:11.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
20:07:11.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 05 20:07:11 dist-test-slave-x5g2 krb5kdc[11415](info): setting up network...
20:07:11.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
20:07:11.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 05 20:07:11 dist-test-slave-x5g2 krb5kdc[11415](info): set up 2 sockets
20:07:11.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 05 20:07:11 dist-test-slave-x5g2 krb5kdc[11415](info): commencing operation
20:07:11.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
20:07:11.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:11.250176 11392 heartbeater.cc:499] Master 127.10.107.189:36917 was elected leader, sending a full tablet report...
20:07:13.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:13.040282 11403 mini_kdc.cc:121] Time spent starting KDC: real 2.043s user 0.005s sys 0.015s
20:07:13.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
20:07:13.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
20:07:13.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
20:07:13.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
20:07:13.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
20:07:13.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
20:07:13.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
20:07:13.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
20:07:13.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
20:07:13.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
20:07:13.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/krb5kdc/test-user.keytab.
20:07:13.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/krb5kdc/test-user.keytab.
20:07:13.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 05 20:07:13 dist-test-slave-x5g2 krb5kdc[11415](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1757102833, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:07:13.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
20:07:13.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-09-05T20:07:13Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
20:07:13.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-09-05T20:07:13Z Disabled control of system clock
20:07:13.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.11.34.254@KRBTEST.COM; defaulting to no policy
20:07:13.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:07:13.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.11.34.254@KRBTEST.COM" created.
20:07:13.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:07:13.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.11.34.254 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/krb5kdc/kudu.keytab.
20:07:13.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.11.34.254 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/krb5kdc/kudu.keytab.
20:07:13.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.11.34.254@KRBTEST.COM; defaulting to no policy
20:07:13.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:07:13.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.11.34.254@KRBTEST.COM" created.
20:07:13.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:07:13.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.11.34.254 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/krb5kdc/kudu.keytab.
20:07:13.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.11.34.254 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/krb5kdc/kudu.keytab.
20:07:13.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:13.259963 11403 external_mini_cluster.cc:1366] Running /tmp/dist-test-taske7evVR/build/tsan/bin/kudu
20:07:13.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taske7evVR/build/tsan/bin/kudu
20:07:13.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/master-0/wal
20:07:13.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/master-0/data
20:07:13.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:07:13.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:07:13.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:07:13.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:07:13.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:07:13.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:07:13.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/master-0/logs
20:07:13.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/master-0/data/info.pb
20:07:13.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:07:13.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:07:13.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:07:13.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:07:13.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:07:13.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:07:13.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:07:13.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:07:13.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:07:13.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:07:13.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:07:13.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:07:13.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.34.254:38675
20:07:13.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.34.254
20:07:13.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:07:13.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/krb5kdc/kudu.keytab
20:07:13.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.11.34.254
20:07:13.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:07:13.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:07:13.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:07:13.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
20:07:13.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.34.212:38019
20:07:13.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:07:13.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:07:13.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:07:13.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:07:13.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.11.34.254:38675,127.11.34.253:41205,127.11.34.252:43017
20:07:13.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:07:13.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34155/jwks.json
20:07:13.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
20:07:13.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:07:13.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:13.569346 11432 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:07:13.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:13.570186 11432 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:07:13.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:13.570928 11432 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:07:13.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:13.606419 11432 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
20:07:13.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:13.606743 11432 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
20:07:13.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:13.607180 11432 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:07:13.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:13.607491 11432 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
20:07:13.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:13.607770 11432 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
20:07:13.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:13.608048 11432 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
20:07:13.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:13.608251 11432 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34155/jwks.json
20:07:13.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:13.647600 11432 master_runner.cc:386] Master server non-default flags:
20:07:13.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:07:13.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.34.212:38019
20:07:13.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:07:13.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:07:13.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/master-0/data
20:07:13.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/master-0/wal
20:07:13.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
20:07:13.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:07:13.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.11.34.254:38675,127.11.34.253:41205,127.11.34.252:43017
20:07:13.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/krb5kdc/kudu.keytab
20:07:13.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.11.34.254
20:07:13.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:07:13.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:07:13.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:07:13.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.34.254:38675
20:07:13.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:07:13.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:07:13.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:07:13.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34155/jwks.json
20:07:13.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:07:13.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:07:13.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:07:13.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:07:13.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:07:13.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/master-0/data/info.pb
20:07:13.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.34.254
20:07:13.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:07:13.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:07:13.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:07:13.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:07:13.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:07:13.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:07:13.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/master-0/logs
20:07:13.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:07:13.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:07:13.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:07:13.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:07:13.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:07:13.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision e2e116a9cb215058aad161061214d11c617cedac
20:07:13.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:07:13.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 05 Sep 2025 19:43:22 UTC on 5fd53c4cbb9d
20:07:13.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7917
20:07:13.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:07:13.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:13.648850 11432 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:07:13.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:13.649838 11432 file_cache.cc:492] Constructed file cache file cache with capacity 419430
20:07:13.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:13.659891 11439 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:07:14.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:13.660691 11440 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:07:14.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:14.958509 11442 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:07:14.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:14.962525 11432 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.301s user 0.382s sys 0.854s
20:07:14.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:14.962893 11432 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.302s user 0.382s sys 0.854s
20:07:14.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:14.963423 11441 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1301 milliseconds
20:07:14.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:14.963461 11432 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:07:14.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:14.965289 11432 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:07:14.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:14.969058 11432 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:07:14.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:14.970536 11432 hybrid_clock.cc:648] HybridClock initialized: now 1757102834970451 us; error 79 us; skew 500 ppm
20:07:14.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:14.971364 11432 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:07:14.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 05 20:07:14 dist-test-slave-x5g2 krb5kdc[11415](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1757102834, etypes {rep=17 tkt=17 ses=17}, kudu/127.11.34.254@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:07:14.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:14.978705 11432 init.cc:377] Logged in from keytab as kudu/127.11.34.254@KRBTEST.COM (short username kudu)
20:07:14.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:14.984035 11432 webserver.cc:480] Webserver started at http://127.11.34.254:44601/ using document root <none> and password file <none>
20:07:14.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:14.984841 11432 fs_manager.cc:362] Metadata directory not provided
20:07:14.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:14.985028 11432 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:07:14.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:14.985395 11432 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
20:07:14.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:14.989763 11432 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/master-0/data/instance:
20:07:14.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e65987b09c81467d8e563fb05fff4cb8"
20:07:14.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-05 20:07:14 on dist-test-slave-x5g2"
20:07:14.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:14.990763 11432 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/master-0/wal/instance:
20:07:14.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e65987b09c81467d8e563fb05fff4cb8"
20:07:14.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-05 20:07:14 on dist-test-slave-x5g2"
20:07:14.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:14.999303 11432 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.007s sys 0.001s
20:07:15.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:15.005060 11450 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:07:15.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:15.006222 11432 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.001s
20:07:15.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:15.006549 11432 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/master-0/data,/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/master-0/wal
20:07:15.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e65987b09c81467d8e563fb05fff4cb8"
20:07:15.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-05 20:07:14 on dist-test-slave-x5g2"
20:07:15.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:15.006860 11432 fs_report.cc:389] FS layout report
20:07:15.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:07:15.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/master-0/wal
20:07:15.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/master-0/wal
20:07:15.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/master-0/data/data
20:07:15.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:07:15.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:07:15.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:07:15.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:07:15.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:07:15.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:07:15.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:07:15.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:07:15.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:07:15.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:07:15.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:07:15.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:15.089857 11432 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:07:15.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:15.095649 11432 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:07:15.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:15.096079 11432 kserver.cc:163] Server-wide thread pool size limit: 3276
20:07:15.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:15.172479 11501 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.11.34.254:38675 every 8 connection(s)
20:07:15.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:15.172463 11432 rpc_server.cc:307] RPC server started. Bound to: 127.11.34.254:38675
20:07:15.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:15.175272 11432 server_base.cc:1179] Dumped server information to /tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/master-0/data/info.pb
20:07:15.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:15.181542 11502 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
20:07:15.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:15.183213 11403 external_mini_cluster.cc:1428] Started /tmp/dist-test-taske7evVR/build/tsan/bin/kudu as pid 11432
20:07:15.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:15.183743 11403 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/master-0/wal/instance
20:07:15.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:15.196789 11502 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.11.34.254" port: 38675 } has no permanent_uuid. Determining permanent_uuid...
20:07:15.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.11.34.253@KRBTEST.COM; defaulting to no policy
20:07:15.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:07:15.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.11.34.253@KRBTEST.COM" created.
20:07:15.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 05 20:07:15 dist-test-slave-x5g2 krb5kdc[11415](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1757102834, etypes {rep=17 tkt=17 ses=17}, kudu/127.11.34.254@KRBTEST.COM for kudu/127.11.34.254@KRBTEST.COM
20:07:15.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:07:15.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.11.34.253 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/krb5kdc/kudu.keytab.
20:07:15.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.11.34.253 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/krb5kdc/kudu.keytab.
20:07:15.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:15.237990 11502 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.11.34.253" port: 41205 } has no permanent_uuid. Determining permanent_uuid...
20:07:15.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:15.240919 11452 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.11.34.253:41205: connect: Connection refused (error 111)
20:07:15.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:15.247237 11502 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.11.34.253:41205: Network error: Client connection negotiation failed: client connection to 127.11.34.253:41205: connect: Connection refused (error 111)
20:07:15.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.11.34.253@KRBTEST.COM; defaulting to no policy
20:07:15.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:07:15.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.11.34.253@KRBTEST.COM" created.
20:07:15.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:07:15.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.11.34.253 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/krb5kdc/kudu.keytab.
20:07:15.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.11.34.253 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/krb5kdc/kudu.keytab.
20:07:15.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:15.279474 11403 external_mini_cluster.cc:1366] Running /tmp/dist-test-taske7evVR/build/tsan/bin/kudu
20:07:15.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taske7evVR/build/tsan/bin/kudu
20:07:15.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/master-1/wal
20:07:15.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/master-1/data
20:07:15.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:07:15.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:07:15.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:07:15.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:07:15.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:07:15.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:07:15.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/master-1/logs
20:07:15.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/master-1/data/info.pb
20:07:15.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:07:15.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:07:15.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:07:15.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:07:15.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:07:15.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:07:15.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:07:15.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:07:15.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:07:15.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:07:15.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:07:15.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:07:15.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.34.253:41205
20:07:15.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.34.253
20:07:15.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:07:15.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/krb5kdc/kudu.keytab
20:07:15.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.11.34.253
20:07:15.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:07:15.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:07:15.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:07:15.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
20:07:15.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.34.212:38019
20:07:15.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:07:15.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:07:15.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:07:15.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:07:15.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.11.34.254:38675,127.11.34.253:41205,127.11.34.252:43017
20:07:15.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:07:15.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34155/jwks.json
20:07:15.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
20:07:15.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:07:15.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:15.296687 11502 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.34.253" port: 41205 } attempt: 1
20:07:15.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:15.300796 11502 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.11.34.253:41205: Network error: Client connection negotiation failed: client connection to 127.11.34.253:41205: connect: Connection refused (error 111)
20:07:15.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:15.369158 11502 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.34.253" port: 41205 } attempt: 2
20:07:15.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:15.372601 11502 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.11.34.253:41205: Network error: Client connection negotiation failed: client connection to 127.11.34.253:41205: connect: Connection refused (error 111)
20:07:15.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:15.463984 11502 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.34.253" port: 41205 } attempt: 3
20:07:15.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:15.468839 11502 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.11.34.253:41205: Network error: Client connection negotiation failed: client connection to 127.11.34.253:41205: connect: Connection refused (error 111)
20:07:15.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:15.600378 11511 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:07:15.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:15.601047 11511 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:07:15.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:15.601821 11511 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:07:15.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:15.612236 11502 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.34.253" port: 41205 } attempt: 4
20:07:15.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:15.616547 11502 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.11.34.253:41205: Network error: Client connection negotiation failed: client connection to 127.11.34.253:41205: connect: Connection refused (error 111)
20:07:15.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:15.635035 11511 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
20:07:15.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:15.635317 11511 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
20:07:15.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:15.635695 11511 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:07:15.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:15.635994 11511 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
20:07:15.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:15.636281 11511 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
20:07:15.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:15.636544 11511 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
20:07:15.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:15.636755 11511 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34155/jwks.json
20:07:15.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:15.674125 11511 master_runner.cc:386] Master server non-default flags:
20:07:15.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:07:15.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.34.212:38019
20:07:15.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:07:15.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:07:15.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/master-1/data
20:07:15.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/master-1/wal
20:07:15.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
20:07:15.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:07:15.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.11.34.254:38675,127.11.34.253:41205,127.11.34.252:43017
20:07:15.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/krb5kdc/kudu.keytab
20:07:15.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.11.34.253
20:07:15.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:07:15.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:07:15.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:07:15.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.34.253:41205
20:07:15.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:07:15.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:07:15.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:07:15.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34155/jwks.json
20:07:15.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:07:15.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:07:15.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:07:15.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:07:15.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:07:15.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/master-1/data/info.pb
20:07:15.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.34.253
20:07:15.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:07:15.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:07:15.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:07:15.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:07:15.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:07:15.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:07:15.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/master-1/logs
20:07:15.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:07:15.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:07:15.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:07:15.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:07:15.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:07:15.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision e2e116a9cb215058aad161061214d11c617cedac
20:07:15.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:07:15.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 05 Sep 2025 19:43:22 UTC on 5fd53c4cbb9d
20:07:15.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7917
20:07:15.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:07:15.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:15.675734 11511 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:07:15.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:15.676851 11511 file_cache.cc:492] Constructed file cache file cache with capacity 419430
20:07:15.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:15.687937 11518 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:07:15.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:15.915974 11502 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.34.253" port: 41205 } attempt: 5
20:07:15.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:15.924225 11502 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.11.34.253:41205: Network error: Client connection negotiation failed: client connection to 127.11.34.253:41205: connect: Connection refused (error 111)
20:07:16.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:16.471788 11502 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.34.253" port: 41205 } attempt: 6
20:07:16.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:16.498492 11502 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.11.34.253:41205: Network error: Client connection negotiation failed: client connection to 127.11.34.253:41205: connect: Connection refused (error 111)
20:07:17.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:17.090898 11517 debug-util.cc:398] Leaking SignalData structure 0x7b0800037200 after lost signal to thread 11511
20:07:17.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:17.328703 11511 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.640s user 0.579s sys 1.050s
20:07:17.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:17.329147 11511 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.641s user 0.579s sys 1.050s
20:07:17.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:15.688812 11519 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:07:17.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:17.330494 11520 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1641 milliseconds
20:07:17.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:17.331015 11521 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:07:17.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:17.331410 11511 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:07:17.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:17.334815 11511 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:07:17.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:17.337775 11511 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:07:17.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:17.339339 11511 hybrid_clock.cc:648] HybridClock initialized: now 1757102837339271 us; error 77 us; skew 500 ppm
20:07:17.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:17.340457 11511 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:07:17.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 05 20:07:17 dist-test-slave-x5g2 krb5kdc[11415](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1757102837, etypes {rep=17 tkt=17 ses=17}, kudu/127.11.34.253@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:07:17.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:17.346556 11511 init.cc:377] Logged in from keytab as kudu/127.11.34.253@KRBTEST.COM (short username kudu)
20:07:17.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:17.350809 11511 webserver.cc:480] Webserver started at http://127.11.34.253:33625/ using document root <none> and password file <none>
20:07:17.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:17.351717 11511 fs_manager.cc:362] Metadata directory not provided
20:07:17.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:17.351934 11511 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:07:17.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:17.352371 11511 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
20:07:17.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:17.356719 11511 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/master-1/data/instance:
20:07:17.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "abed6b5ab4604caa92637633d9f26bbf"
20:07:17.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-05 20:07:17 on dist-test-slave-x5g2"
20:07:17.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:17.357798 11511 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/master-1/wal/instance:
20:07:17.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "abed6b5ab4604caa92637633d9f26bbf"
20:07:17.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-05 20:07:17 on dist-test-slave-x5g2"
20:07:17.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:17.364571 11511 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.006s sys 0.000s
20:07:17.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:17.370119 11538 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:07:17.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:17.371112 11511 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.004s sys 0.000s
20:07:17.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:17.371466 11511 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/master-1/data,/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/master-1/wal
20:07:17.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "abed6b5ab4604caa92637633d9f26bbf"
20:07:17.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-05 20:07:17 on dist-test-slave-x5g2"
20:07:17.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:17.372112 11511 fs_report.cc:389] FS layout report
20:07:17.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:07:17.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/master-1/wal
20:07:17.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/master-1/wal
20:07:17.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/master-1/data/data
20:07:17.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:07:17.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:07:17.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:07:17.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:07:17.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:07:17.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:07:17.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:07:17.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:07:17.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:07:17.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:07:17.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:07:17.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:17.434937 11511 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:07:17.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:17.440486 11511 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:07:17.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:17.440963 11511 kserver.cc:163] Server-wide thread pool size limit: 3276
20:07:17.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:17.525123 11511 rpc_server.cc:307] RPC server started. Bound to: 127.11.34.253:41205
20:07:17.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:17.525192 11589 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.11.34.253:41205 every 8 connection(s)
20:07:17.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:17.527777 11511 server_base.cc:1179] Dumped server information to /tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/master-1/data/info.pb
20:07:17.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:17.529429 11403 external_mini_cluster.cc:1428] Started /tmp/dist-test-taske7evVR/build/tsan/bin/kudu as pid 11511
20:07:17.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:17.529839 11403 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/master-1/wal/instance
20:07:17.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:17.535269 11590 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
20:07:17.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.11.34.252@KRBTEST.COM; defaulting to no policy
20:07:17.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:07:17.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.11.34.252@KRBTEST.COM" created.
20:07:17.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:17.546092 11590 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.11.34.254" port: 38675 } has no permanent_uuid. Determining permanent_uuid...
20:07:17.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:17.559032 11502 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.34.253" port: 41205 } attempt: 7
20:07:17.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 05 20:07:17 dist-test-slave-x5g2 krb5kdc[11415](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1757102837, etypes {rep=17 tkt=17 ses=17}, kudu/127.11.34.253@KRBTEST.COM for kudu/127.11.34.254@KRBTEST.COM
20:07:17.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:07:17.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.11.34.252 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/krb5kdc/kudu.keytab.
20:07:17.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.11.34.252 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/krb5kdc/kudu.keytab.
20:07:17.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 05 20:07:17 dist-test-slave-x5g2 krb5kdc[11415](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1757102834, etypes {rep=17 tkt=17 ses=17}, kudu/127.11.34.254@KRBTEST.COM for kudu/127.11.34.253@KRBTEST.COM
20:07:17.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:17.582952 11590 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.11.34.253" port: 41205 } has no permanent_uuid. Determining permanent_uuid...
20:07:17.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.11.34.252@KRBTEST.COM; defaulting to no policy
20:07:17.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:17.592518 11502 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.11.34.252" port: 43017 } has no permanent_uuid. Determining permanent_uuid...
20:07:17.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:07:17.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.11.34.252@KRBTEST.COM" created.
20:07:17.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 05 20:07:17 dist-test-slave-x5g2 krb5kdc[11415](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1757102837, etypes {rep=17 tkt=17 ses=17}, kudu/127.11.34.253@KRBTEST.COM for kudu/127.11.34.253@KRBTEST.COM
20:07:17.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:17.608759 11502 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.11.34.252:43017: Network error: Client connection negotiation failed: client connection to 127.11.34.252:43017: connect: Connection refused (error 111)
20:07:17.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:17.611724 11590 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.11.34.252" port: 43017 } has no permanent_uuid. Determining permanent_uuid...
20:07:17.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:17.613860 11540 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.11.34.252:43017: connect: Connection refused (error 111)
20:07:17.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:17.618641 11590 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.11.34.252:43017: Network error: Client connection negotiation failed: client connection to 127.11.34.252:43017: connect: Connection refused (error 111)
20:07:17.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:07:17.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.11.34.252 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/krb5kdc/kudu.keytab.
20:07:17.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.11.34.252 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/krb5kdc/kudu.keytab.
20:07:17.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:17.621723 11403 external_mini_cluster.cc:1366] Running /tmp/dist-test-taske7evVR/build/tsan/bin/kudu
20:07:17.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taske7evVR/build/tsan/bin/kudu
20:07:17.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/master-2/wal
20:07:17.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/master-2/data
20:07:17.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:07:17.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:07:17.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:07:17.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:07:17.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:07:17.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:07:17.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/master-2/logs
20:07:17.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/master-2/data/info.pb
20:07:17.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:07:17.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:07:17.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:07:17.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:07:17.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:07:17.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:07:17.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:07:17.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:07:17.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:07:17.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:07:17.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:07:17.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:07:17.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.34.252:43017
20:07:17.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.34.252
20:07:17.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:07:17.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/krb5kdc/kudu.keytab
20:07:17.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.11.34.252
20:07:17.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:07:17.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:07:17.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:07:17.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
20:07:17.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.34.212:38019
20:07:17.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:07:17.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:07:17.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:07:17.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:07:17.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.11.34.254:38675,127.11.34.253:41205,127.11.34.252:43017
20:07:17.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:07:17.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34155/jwks.json
20:07:17.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
20:07:17.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:07:17.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:17.667204 11502 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.34.252" port: 43017 } attempt: 1
20:07:17.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:17.668236 11590 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.34.252" port: 43017 } attempt: 1
20:07:17.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:17.672221 11502 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.11.34.252:43017: Network error: Client connection negotiation failed: client connection to 127.11.34.252:43017: connect: Connection refused (error 111)
20:07:17.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:17.672976 11590 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.11.34.252:43017: Network error: Client connection negotiation failed: client connection to 127.11.34.252:43017: connect: Connection refused (error 111)
20:07:17.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:17.741340 11590 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.34.252" port: 43017 } attempt: 2
20:07:17.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:17.745364 11590 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.11.34.252:43017: Network error: Client connection negotiation failed: client connection to 127.11.34.252:43017: connect: Connection refused (error 111)
20:07:17.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:17.753615 11502 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.34.252" port: 43017 } attempt: 2
20:07:17.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:17.757022 11502 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.11.34.252:43017: Network error: Client connection negotiation failed: client connection to 127.11.34.252:43017: connect: Connection refused (error 111)
20:07:17.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:17.836791 11590 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.34.252" port: 43017 } attempt: 3
20:07:17.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:17.840803 11590 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.11.34.252:43017: Network error: Client connection negotiation failed: client connection to 127.11.34.252:43017: connect: Connection refused (error 111)
20:07:17.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:17.842430 11502 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.34.252" port: 43017 } attempt: 3
20:07:17.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:17.846043 11502 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.11.34.252:43017: Network error: Client connection negotiation failed: client connection to 127.11.34.252:43017: connect: Connection refused (error 111)
20:07:17.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:17.928939 11603 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:07:17.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:17.929512 11603 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:07:17.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:17.930213 11603 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:07:17.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:17.960788 11603 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
20:07:17.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:17.961025 11603 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
20:07:17.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:17.961292 11603 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:07:17.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:17.961504 11603 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
20:07:17.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:17.961689 11603 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
20:07:17.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:17.961898 11603 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
20:07:17.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:17.962038 11603 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34155/jwks.json
20:07:17.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:17.984247 11590 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.34.252" port: 43017 } attempt: 4
20:07:17.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:17.986459 11502 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.34.252" port: 43017 } attempt: 4
20:07:17.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:17.988657 11590 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.11.34.252:43017: Network error: Client connection negotiation failed: client connection to 127.11.34.252:43017: connect: Connection refused (error 111)
20:07:17.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:17.990110 11502 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.11.34.252:43017: Network error: Client connection negotiation failed: client connection to 127.11.34.252:43017: connect: Connection refused (error 111)
20:07:18.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:18.001511 11603 master_runner.cc:386] Master server non-default flags:
20:07:18.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:07:18.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.34.212:38019
20:07:18.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:07:18.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:07:18.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/master-2/data
20:07:18.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/master-2/wal
20:07:18.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
20:07:18.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:07:18.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.11.34.254:38675,127.11.34.253:41205,127.11.34.252:43017
20:07:18.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/krb5kdc/kudu.keytab
20:07:18.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.11.34.252
20:07:18.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:07:18.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:07:18.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:07:18.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.34.252:43017
20:07:18.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:07:18.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:07:18.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:07:18.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34155/jwks.json
20:07:18.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:07:18.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:07:18.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:07:18.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:07:18.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:07:18.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/master-2/data/info.pb
20:07:18.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.34.252
20:07:18.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:07:18.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:07:18.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:07:18.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:07:18.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:07:18.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:07:18.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/master-2/logs
20:07:18.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:07:18.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:07:18.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:07:18.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:07:18.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:07:18.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision e2e116a9cb215058aad161061214d11c617cedac
20:07:18.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:07:18.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 05 Sep 2025 19:43:22 UTC on 5fd53c4cbb9d
20:07:18.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7917
20:07:18.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:07:18.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:18.002763 11603 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:07:18.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:18.004166 11603 file_cache.cc:492] Constructed file cache file cache with capacity 419430
20:07:18.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:18.018508 11609 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:07:18.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:18.273484 11502 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.34.252" port: 43017 } attempt: 5
20:07:18.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:18.278892 11502 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.11.34.252:43017: Network error: Client connection negotiation failed: client connection to 127.11.34.252:43017: connect: Connection refused (error 111)
20:07:18.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:18.288173 11590 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.34.252" port: 43017 } attempt: 5
20:07:18.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:18.292470 11590 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.11.34.252:43017: Network error: Client connection negotiation failed: client connection to 127.11.34.252:43017: connect: Connection refused (error 111)
20:07:18.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:18.831321 11502 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.34.252" port: 43017 } attempt: 6
20:07:18.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:18.840142 11590 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.34.252" port: 43017 } attempt: 6
20:07:18.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:18.843364 11502 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.11.34.252:43017: Network error: Client connection negotiation failed: client connection to 127.11.34.252:43017: connect: Connection refused (error 111)
20:07:18.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:18.853128 11590 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.11.34.252:43017: Network error: Client connection negotiation failed: client connection to 127.11.34.252:43017: connect: Connection refused (error 111)
20:07:19.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:18.025251 11612 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:07:19.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:18.021283 11610 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:07:19.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:19.375756 11611 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1352 milliseconds
20:07:19.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:19.375888 11603 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:07:19.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:19.377049 11603 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:07:19.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:19.380752 11603 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:07:19.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:19.382705 11603 hybrid_clock.cc:648] HybridClock initialized: now 1757102839382615 us; error 92 us; skew 500 ppm
20:07:19.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:19.383603 11603 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:07:19.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 05 20:07:19 dist-test-slave-x5g2 krb5kdc[11415](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1757102839, etypes {rep=17 tkt=17 ses=17}, kudu/127.11.34.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:07:19.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:19.389222 11603 init.cc:377] Logged in from keytab as kudu/127.11.34.252@KRBTEST.COM (short username kudu)
20:07:19.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:19.394322 11603 webserver.cc:480] Webserver started at http://127.11.34.252:38995/ using document root <none> and password file <none>
20:07:19.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:19.395159 11603 fs_manager.cc:362] Metadata directory not provided
20:07:19.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:19.395335 11603 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:07:19.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:19.395714 11603 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
20:07:19.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:19.400198 11603 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/master-2/data/instance:
20:07:19.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "215376717c434b229999d7279f7e12a6"
20:07:19.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-05 20:07:19 on dist-test-slave-x5g2"
20:07:19.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:19.401283 11603 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/master-2/wal/instance:
20:07:19.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "215376717c434b229999d7279f7e12a6"
20:07:19.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-05 20:07:19 on dist-test-slave-x5g2"
20:07:19.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:19.409775 11603 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.007s sys 0.001s
20:07:19.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:19.416252 11626 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:07:19.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:19.417279 11603 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.006s sys 0.000s
20:07:19.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:19.417590 11603 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/master-2/data,/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/master-2/wal
20:07:19.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "215376717c434b229999d7279f7e12a6"
20:07:19.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-05 20:07:19 on dist-test-slave-x5g2"
20:07:19.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:19.417883 11603 fs_report.cc:389] FS layout report
20:07:19.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:07:19.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/master-2/wal
20:07:19.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/master-2/wal
20:07:19.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/master-2/data/data
20:07:19.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:07:19.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:07:19.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:07:19.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:07:19.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:07:19.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:07:19.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:07:19.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:07:19.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:07:19.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:07:19.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:07:19.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:19.474114 11603 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:07:19.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:19.484812 11603 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:07:19.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:19.485309 11603 kserver.cc:163] Server-wide thread pool size limit: 3276
20:07:19.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:19.566689 11603 rpc_server.cc:307] RPC server started. Bound to: 127.11.34.252:43017
20:07:19.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:19.566769 11677 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.11.34.252:43017 every 8 connection(s)
20:07:19.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:19.569337 11603 server_base.cc:1179] Dumped server information to /tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/master-2/data/info.pb
20:07:19.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:19.571695 11403 external_mini_cluster.cc:1428] Started /tmp/dist-test-taske7evVR/build/tsan/bin/kudu as pid 11603
20:07:19.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:19.572139 11403 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/master-2/wal/instance
20:07:19.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:19.576674 11678 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
20:07:19.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:19.594851 11678 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.11.34.254" port: 38675 } has no permanent_uuid. Determining permanent_uuid...
20:07:19.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 05 20:07:19 dist-test-slave-x5g2 krb5kdc[11415](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1757102833, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.11.34.254@KRBTEST.COM
20:07:19.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 05 20:07:19 dist-test-slave-x5g2 krb5kdc[11415](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1757102839, etypes {rep=17 tkt=17 ses=17}, kudu/127.11.34.252@KRBTEST.COM for kudu/127.11.34.254@KRBTEST.COM
20:07:19.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 05 20:07:19 dist-test-slave-x5g2 krb5kdc[11415](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1757102833, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.11.34.253@KRBTEST.COM
20:07:19.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:19.642182 11678 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.11.34.253" port: 41205 } has no permanent_uuid. Determining permanent_uuid...
20:07:19.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 05 20:07:19 dist-test-slave-x5g2 krb5kdc[11415](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1757102839, etypes {rep=17 tkt=17 ses=17}, kudu/127.11.34.252@KRBTEST.COM for kudu/127.11.34.253@KRBTEST.COM
20:07:19.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 05 20:07:19 dist-test-slave-x5g2 krb5kdc[11415](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1757102833, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.11.34.252@KRBTEST.COM
20:07:19.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:19.663975 11678 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.11.34.252" port: 43017 } has no permanent_uuid. Determining permanent_uuid...
20:07:19.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 05 20:07:19 dist-test-slave-x5g2 krb5kdc[11415](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1757102839, etypes {rep=17 tkt=17 ses=17}, kudu/127.11.34.252@KRBTEST.COM for kudu/127.11.34.252@KRBTEST.COM
20:07:19.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:19.694980 11678 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 215376717c434b229999d7279f7e12a6: Bootstrap starting.
20:07:19.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:19.701782 11678 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 215376717c434b229999d7279f7e12a6: Neither blocks nor log segments found. Creating new log.
20:07:19.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:19.704247 11678 log.cc:826] T 00000000000000000000000000000000 P 215376717c434b229999d7279f7e12a6: Log is configured to *not* fsync() on all Append() calls
20:07:19.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:19.709015 11678 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 215376717c434b229999d7279f7e12a6: No bootstrap required, opened a new log
20:07:19.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:19.729504 11678 raft_consensus.cc:357] T 00000000000000000000000000000000 P 215376717c434b229999d7279f7e12a6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e65987b09c81467d8e563fb05fff4cb8" member_type: VOTER last_known_addr { host: "127.11.34.254" port: 38675 } } peers { permanent_uuid: "abed6b5ab4604caa92637633d9f26bbf" member_type: VOTER last_known_addr { host: "127.11.34.253" port: 41205 } } peers { permanent_uuid: "215376717c434b229999d7279f7e12a6" member_type: VOTER last_known_addr { host: "127.11.34.252" port: 43017 } }
20:07:19.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:19.730283 11678 raft_consensus.cc:383] T 00000000000000000000000000000000 P 215376717c434b229999d7279f7e12a6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:07:19.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:19.730494 11678 raft_consensus.cc:738] T 00000000000000000000000000000000 P 215376717c434b229999d7279f7e12a6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 215376717c434b229999d7279f7e12a6, State: Initialized, Role: FOLLOWER
20:07:19.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:19.731242 11678 consensus_queue.cc:260] T 00000000000000000000000000000000 P 215376717c434b229999d7279f7e12a6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e65987b09c81467d8e563fb05fff4cb8" member_type: VOTER last_known_addr { host: "127.11.34.254" port: 38675 } } peers { permanent_uuid: "abed6b5ab4604caa92637633d9f26bbf" member_type: VOTER last_known_addr { host: "127.11.34.253" port: 41205 } } peers { permanent_uuid: "215376717c434b229999d7279f7e12a6" member_type: VOTER last_known_addr { host: "127.11.34.252" port: 43017 } }
20:07:19.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:19.734745 11688 sys_catalog.cc:455] T 00000000000000000000000000000000 P 215376717c434b229999d7279f7e12a6 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e65987b09c81467d8e563fb05fff4cb8" member_type: VOTER last_known_addr { host: "127.11.34.254" port: 38675 } } peers { permanent_uuid: "abed6b5ab4604caa92637633d9f26bbf" member_type: VOTER last_known_addr { host: "127.11.34.253" port: 41205 } } peers { permanent_uuid: "215376717c434b229999d7279f7e12a6" member_type: VOTER last_known_addr { host: "127.11.34.252" port: 43017 } } }
20:07:19.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:19.735481 11688 sys_catalog.cc:458] T 00000000000000000000000000000000 P 215376717c434b229999d7279f7e12a6 [sys.catalog]: This master's current role is: FOLLOWER
20:07:19.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:19.736470 11678 sys_catalog.cc:564] T 00000000000000000000000000000000 P 215376717c434b229999d7279f7e12a6 [sys.catalog]: configured and running, proceeding with master startup.
20:07:19.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:19.741236 11632 tablet.cc:2378] T 00000000000000000000000000000000 P 215376717c434b229999d7279f7e12a6: Can't schedule compaction. Clean time has not been advanced past its initial value.
20:07:19.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:19.774453 11699 catalog_manager.cc:1560] T 00000000000000000000000000000000 P 215376717c434b229999d7279f7e12a6: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:07:19.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:19.774766 11699 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:07:19.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:19.760649 11603 master_runner.cc:186] Error getting master registration for 127.11.34.254:38675: OK, instance_id { permanent_uuid: "e65987b09c81467d8e563fb05fff4cb8" instance_seqno: 1757102835149886 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
20:07:19.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:19.793812 11603 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:07:19.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:19.876863 11502 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.34.252" port: 43017 } attempt: 7
20:07:19.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 05 20:07:19 dist-test-slave-x5g2 krb5kdc[11415](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1757102834, etypes {rep=17 tkt=17 ses=17}, kudu/127.11.34.254@KRBTEST.COM for kudu/127.11.34.252@KRBTEST.COM
20:07:19.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:19.891608 11688 raft_consensus.cc:491] T 00000000000000000000000000000000 P 215376717c434b229999d7279f7e12a6 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:07:19.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:19.892722 11688 raft_consensus.cc:513] T 00000000000000000000000000000000 P 215376717c434b229999d7279f7e12a6 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e65987b09c81467d8e563fb05fff4cb8" member_type: VOTER last_known_addr { host: "127.11.34.254" port: 38675 } } peers { permanent_uuid: "abed6b5ab4604caa92637633d9f26bbf" member_type: VOTER last_known_addr { host: "127.11.34.253" port: 41205 } } peers { permanent_uuid: "215376717c434b229999d7279f7e12a6" member_type: VOTER last_known_addr { host: "127.11.34.252" port: 43017 } }
20:07:19.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:19.899508 11688 leader_election.cc:290] T 00000000000000000000000000000000 P 215376717c434b229999d7279f7e12a6 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers e65987b09c81467d8e563fb05fff4cb8 (127.11.34.254:38675), abed6b5ab4604caa92637633d9f26bbf (127.11.34.253:41205)
20:07:19.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:19.910555 11502 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P e65987b09c81467d8e563fb05fff4cb8: Bootstrap starting.
20:07:19.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:19.913659 11590 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.34.252" port: 43017 } attempt: 7
20:07:19.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:19.919170 11502 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P e65987b09c81467d8e563fb05fff4cb8: Neither blocks nor log segments found. Creating new log.
20:07:19.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:19.922456 11502 log.cc:826] T 00000000000000000000000000000000 P e65987b09c81467d8e563fb05fff4cb8: Log is configured to *not* fsync() on all Append() calls
20:07:19.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 05 20:07:19 dist-test-slave-x5g2 krb5kdc[11415](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1757102837, etypes {rep=17 tkt=17 ses=17}, kudu/127.11.34.253@KRBTEST.COM for kudu/127.11.34.252@KRBTEST.COM
20:07:19.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:19.934505 11502 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P e65987b09c81467d8e563fb05fff4cb8: No bootstrap required, opened a new log
20:07:19.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:19.956089 11590 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P abed6b5ab4604caa92637633d9f26bbf: Bootstrap starting.
20:07:19.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:19.964990 11590 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P abed6b5ab4604caa92637633d9f26bbf: Neither blocks nor log segments found. Creating new log.
20:07:19.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:19.967813 11590 log.cc:826] T 00000000000000000000000000000000 P abed6b5ab4604caa92637633d9f26bbf: Log is configured to *not* fsync() on all Append() calls
20:07:19.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:19.970549 11502 raft_consensus.cc:357] T 00000000000000000000000000000000 P e65987b09c81467d8e563fb05fff4cb8 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e65987b09c81467d8e563fb05fff4cb8" member_type: VOTER last_known_addr { host: "127.11.34.254" port: 38675 } } peers { permanent_uuid: "abed6b5ab4604caa92637633d9f26bbf" member_type: VOTER last_known_addr { host: "127.11.34.253" port: 41205 } } peers { permanent_uuid: "215376717c434b229999d7279f7e12a6" member_type: VOTER last_known_addr { host: "127.11.34.252" port: 43017 } }
20:07:19.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:19.971410 11502 raft_consensus.cc:383] T 00000000000000000000000000000000 P e65987b09c81467d8e563fb05fff4cb8 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:07:19.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:19.971683 11502 raft_consensus.cc:738] T 00000000000000000000000000000000 P e65987b09c81467d8e563fb05fff4cb8 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e65987b09c81467d8e563fb05fff4cb8, State: Initialized, Role: FOLLOWER
20:07:19.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:19.972532 11502 consensus_queue.cc:260] T 00000000000000000000000000000000 P e65987b09c81467d8e563fb05fff4cb8 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e65987b09c81467d8e563fb05fff4cb8" member_type: VOTER last_known_addr { host: "127.11.34.254" port: 38675 } } peers { permanent_uuid: "abed6b5ab4604caa92637633d9f26bbf" member_type: VOTER last_known_addr { host: "127.11.34.253" port: 41205 } } peers { permanent_uuid: "215376717c434b229999d7279f7e12a6" member_type: VOTER last_known_addr { host: "127.11.34.252" port: 43017 } }
20:07:19.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:19.975558 11705 sys_catalog.cc:455] T 00000000000000000000000000000000 P e65987b09c81467d8e563fb05fff4cb8 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e65987b09c81467d8e563fb05fff4cb8" member_type: VOTER last_known_addr { host: "127.11.34.254" port: 38675 } } peers { permanent_uuid: "abed6b5ab4604caa92637633d9f26bbf" member_type: VOTER last_known_addr { host: "127.11.34.253" port: 41205 } } peers { permanent_uuid: "215376717c434b229999d7279f7e12a6" member_type: VOTER last_known_addr { host: "127.11.34.252" port: 43017 } } }
20:07:19.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:19.977123 11705 sys_catalog.cc:458] T 00000000000000000000000000000000 P e65987b09c81467d8e563fb05fff4cb8 [sys.catalog]: This master's current role is: FOLLOWER
20:07:19.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:19.978579 11590 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P abed6b5ab4604caa92637633d9f26bbf: No bootstrap required, opened a new log
20:07:19.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:19.978749 11502 sys_catalog.cc:564] T 00000000000000000000000000000000 P e65987b09c81467d8e563fb05fff4cb8 [sys.catalog]: configured and running, proceeding with master startup.
20:07:19.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:19.978893 11477 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "215376717c434b229999d7279f7e12a6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e65987b09c81467d8e563fb05fff4cb8" is_pre_election: true
20:07:19.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:19.979746 11477 raft_consensus.cc:2466] T 00000000000000000000000000000000 P e65987b09c81467d8e563fb05fff4cb8 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 215376717c434b229999d7279f7e12a6 in term 0.
20:07:19.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:19.984576 11628 leader_election.cc:304] T 00000000000000000000000000000000 P 215376717c434b229999d7279f7e12a6 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 215376717c434b229999d7279f7e12a6, e65987b09c81467d8e563fb05fff4cb8; no voters:
20:07:19.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:19.988381 11688 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 215376717c434b229999d7279f7e12a6 [term 0 FOLLOWER]: Leader pre-election won for term 1
20:07:19.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:19.988744 11688 raft_consensus.cc:491] T 00000000000000000000000000000000 P 215376717c434b229999d7279f7e12a6 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
20:07:19.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:19.989038 11688 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 215376717c434b229999d7279f7e12a6 [term 0 FOLLOWER]: Advancing to term 1
20:07:20.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:19.998776 11688 raft_consensus.cc:513] T 00000000000000000000000000000000 P 215376717c434b229999d7279f7e12a6 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e65987b09c81467d8e563fb05fff4cb8" member_type: VOTER last_known_addr { host: "127.11.34.254" port: 38675 } } peers { permanent_uuid: "abed6b5ab4604caa92637633d9f26bbf" member_type: VOTER last_known_addr { host: "127.11.34.253" port: 41205 } } peers { permanent_uuid: "215376717c434b229999d7279f7e12a6" member_type: VOTER last_known_addr { host: "127.11.34.252" port: 43017 } }
20:07:20.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:20.002259 11477 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "215376717c434b229999d7279f7e12a6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e65987b09c81467d8e563fb05fff4cb8"
20:07:20.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:20.002853 11477 raft_consensus.cc:3058] T 00000000000000000000000000000000 P e65987b09c81467d8e563fb05fff4cb8 [term 0 FOLLOWER]: Advancing to term 1
20:07:20.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:20.012730 11477 raft_consensus.cc:2466] T 00000000000000000000000000000000 P e65987b09c81467d8e563fb05fff4cb8 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 215376717c434b229999d7279f7e12a6 in term 1.
20:07:20.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:20.013396 11688 leader_election.cc:290] T 00000000000000000000000000000000 P 215376717c434b229999d7279f7e12a6 [CANDIDATE]: Term 1 election: Requested vote from peers e65987b09c81467d8e563fb05fff4cb8 (127.11.34.254:38675), abed6b5ab4604caa92637633d9f26bbf (127.11.34.253:41205)
20:07:20.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:20.014600 11628 leader_election.cc:304] T 00000000000000000000000000000000 P 215376717c434b229999d7279f7e12a6 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 215376717c434b229999d7279f7e12a6, e65987b09c81467d8e563fb05fff4cb8; no voters:
20:07:20.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:20.024251 11688 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 215376717c434b229999d7279f7e12a6 [term 1 FOLLOWER]: Leader election won for term 1
20:07:20.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:20.024750 11688 raft_consensus.cc:695] T 00000000000000000000000000000000 P 215376717c434b229999d7279f7e12a6 [term 1 LEADER]: Becoming Leader. State: Replica: 215376717c434b229999d7279f7e12a6, State: Running, Role: LEADER
20:07:20.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:20.025589 11688 consensus_queue.cc:237] T 00000000000000000000000000000000 P 215376717c434b229999d7279f7e12a6 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e65987b09c81467d8e563fb05fff4cb8" member_type: VOTER last_known_addr { host: "127.11.34.254" port: 38675 } } peers { permanent_uuid: "abed6b5ab4604caa92637633d9f26bbf" member_type: VOTER last_known_addr { host: "127.11.34.253" port: 41205 } } peers { permanent_uuid: "215376717c434b229999d7279f7e12a6" member_type: VOTER last_known_addr { host: "127.11.34.252" port: 43017 } }
20:07:20.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:20.047365 11710 sys_catalog.cc:455] T 00000000000000000000000000000000 P 215376717c434b229999d7279f7e12a6 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 215376717c434b229999d7279f7e12a6. Latest consensus state: current_term: 1 leader_uuid: "215376717c434b229999d7279f7e12a6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e65987b09c81467d8e563fb05fff4cb8" member_type: VOTER last_known_addr { host: "127.11.34.254" port: 38675 } } peers { permanent_uuid: "abed6b5ab4604caa92637633d9f26bbf" member_type: VOTER last_known_addr { host: "127.11.34.253" port: 41205 } } peers { permanent_uuid: "215376717c434b229999d7279f7e12a6" member_type: VOTER last_known_addr { host: "127.11.34.252" port: 43017 } } }
20:07:20.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:20.050554 11710 sys_catalog.cc:458] T 00000000000000000000000000000000 P 215376717c434b229999d7279f7e12a6 [sys.catalog]: This master's current role is: LEADER
20:07:20.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:20.050704 11590 raft_consensus.cc:357] T 00000000000000000000000000000000 P abed6b5ab4604caa92637633d9f26bbf [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e65987b09c81467d8e563fb05fff4cb8" member_type: VOTER last_known_addr { host: "127.11.34.254" port: 38675 } } peers { permanent_uuid: "abed6b5ab4604caa92637633d9f26bbf" member_type: VOTER last_known_addr { host: "127.11.34.253" port: 41205 } } peers { permanent_uuid: "215376717c434b229999d7279f7e12a6" member_type: VOTER last_known_addr { host: "127.11.34.252" port: 43017 } }
20:07:20.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:20.053000 11590 raft_consensus.cc:383] T 00000000000000000000000000000000 P abed6b5ab4604caa92637633d9f26bbf [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:07:20.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:20.053836 11590 raft_consensus.cc:738] T 00000000000000000000000000000000 P abed6b5ab4604caa92637633d9f26bbf [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: abed6b5ab4604caa92637633d9f26bbf, State: Initialized, Role: FOLLOWER
20:07:20.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:20.056401 11716 catalog_manager.cc:1477] Loading table and tablet metadata into memory...
20:07:20.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:20.056342 11590 consensus_queue.cc:260] T 00000000000000000000000000000000 P abed6b5ab4604caa92637633d9f26bbf [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e65987b09c81467d8e563fb05fff4cb8" member_type: VOTER last_known_addr { host: "127.11.34.254" port: 38675 } } peers { permanent_uuid: "abed6b5ab4604caa92637633d9f26bbf" member_type: VOTER last_known_addr { host: "127.11.34.253" port: 41205 } } peers { permanent_uuid: "215376717c434b229999d7279f7e12a6" member_type: VOTER last_known_addr { host: "127.11.34.252" port: 43017 } }
20:07:20.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:20.065305 11716 catalog_manager.cc:1486] Initializing Kudu cluster ID...
20:07:20.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:20.078900 11564 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "215376717c434b229999d7279f7e12a6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "abed6b5ab4604caa92637633d9f26bbf"
20:07:20.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:20.079504 11720 catalog_manager.cc:797] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
20:07:20.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:20.079751 11564 raft_consensus.cc:3058] T 00000000000000000000000000000000 P abed6b5ab4604caa92637633d9f26bbf [term 0 FOLLOWER]: Advancing to term 1
20:07:20.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:20.082033 11565 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "215376717c434b229999d7279f7e12a6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "abed6b5ab4604caa92637633d9f26bbf" is_pre_election: true
20:07:20.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:20.091897 11564 raft_consensus.cc:2466] T 00000000000000000000000000000000 P abed6b5ab4604caa92637633d9f26bbf [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 215376717c434b229999d7279f7e12a6 in term 1.
20:07:20.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:20.092171 11477 raft_consensus.cc:1273] T 00000000000000000000000000000000 P e65987b09c81467d8e563fb05fff4cb8 [term 1 FOLLOWER]: Refusing update from remote peer 215376717c434b229999d7279f7e12a6: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
20:07:20.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:20.092993 11565 raft_consensus.cc:1273] T 00000000000000000000000000000000 P abed6b5ab4604caa92637633d9f26bbf [term 1 FOLLOWER]: Refusing update from remote peer 215376717c434b229999d7279f7e12a6: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
20:07:20.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:20.092717 11718 sys_catalog.cc:455] T 00000000000000000000000000000000 P abed6b5ab4604caa92637633d9f26bbf [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e65987b09c81467d8e563fb05fff4cb8" member_type: VOTER last_known_addr { host: "127.11.34.254" port: 38675 } } peers { permanent_uuid: "abed6b5ab4604caa92637633d9f26bbf" member_type: VOTER last_known_addr { host: "127.11.34.253" port: 41205 } } peers { permanent_uuid: "215376717c434b229999d7279f7e12a6" member_type: VOTER last_known_addr { host: "127.11.34.252" port: 43017 } } }
20:07:20.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:20.093576 11718 sys_catalog.cc:458] T 00000000000000000000000000000000 P abed6b5ab4604caa92637633d9f26bbf [sys.catalog]: This master's current role is: FOLLOWER
20:07:20.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:20.094153 11590 sys_catalog.cc:564] T 00000000000000000000000000000000 P abed6b5ab4604caa92637633d9f26bbf [sys.catalog]: configured and running, proceeding with master startup.
20:07:20.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:20.093844 11710 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 215376717c434b229999d7279f7e12a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e65987b09c81467d8e563fb05fff4cb8" member_type: VOTER last_known_addr { host: "127.11.34.254" port: 38675 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
20:07:20.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:20.101766 11710 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 215376717c434b229999d7279f7e12a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "abed6b5ab4604caa92637633d9f26bbf" member_type: VOTER last_known_addr { host: "127.11.34.253" port: 41205 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
20:07:20.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:20.109076 11456 tablet.cc:2378] T 00000000000000000000000000000000 P e65987b09c81467d8e563fb05fff4cb8: Can't schedule compaction. Clean time has not been advanced past its initial value.
20:07:20.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:20.138309 11718 sys_catalog.cc:455] T 00000000000000000000000000000000 P abed6b5ab4604caa92637633d9f26bbf [sys.catalog]: SysCatalogTable state changed. Reason: New leader 215376717c434b229999d7279f7e12a6. Latest consensus state: current_term: 1 leader_uuid: "215376717c434b229999d7279f7e12a6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e65987b09c81467d8e563fb05fff4cb8" member_type: VOTER last_known_addr { host: "127.11.34.254" port: 38675 } } peers { permanent_uuid: "abed6b5ab4604caa92637633d9f26bbf" member_type: VOTER last_known_addr { host: "127.11.34.253" port: 41205 } } peers { permanent_uuid: "215376717c434b229999d7279f7e12a6" member_type: VOTER last_known_addr { host: "127.11.34.252" port: 43017 } } }
20:07:20.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:20.139835 11718 sys_catalog.cc:458] T 00000000000000000000000000000000 P abed6b5ab4604caa92637633d9f26bbf [sys.catalog]: This master's current role is: FOLLOWER
20:07:20.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:20.145110 11721 mvcc.cc:204] Tried to move back new op lower bound from 7197093232989335552 to 7197093232749965312. Current Snapshot: MvccSnapshot[applied={T|T < 7197093232989335552}]
20:07:20.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:20.150509 11723 mvcc.cc:204] Tried to move back new op lower bound from 7197093232989335552 to 7197093232749965312. Current Snapshot: MvccSnapshot[applied={T|T < 7197093232989335552}]
20:07:20.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:20.167289 11705 sys_catalog.cc:455] T 00000000000000000000000000000000 P e65987b09c81467d8e563fb05fff4cb8 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 215376717c434b229999d7279f7e12a6. Latest consensus state: current_term: 1 leader_uuid: "215376717c434b229999d7279f7e12a6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e65987b09c81467d8e563fb05fff4cb8" member_type: VOTER last_known_addr { host: "127.11.34.254" port: 38675 } } peers { permanent_uuid: "abed6b5ab4604caa92637633d9f26bbf" member_type: VOTER last_known_addr { host: "127.11.34.253" port: 41205 } } peers { permanent_uuid: "215376717c434b229999d7279f7e12a6" member_type: VOTER last_known_addr { host: "127.11.34.252" port: 43017 } } }
20:07:20.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:20.167670 11718 sys_catalog.cc:455] T 00000000000000000000000000000000 P abed6b5ab4604caa92637633d9f26bbf [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "215376717c434b229999d7279f7e12a6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e65987b09c81467d8e563fb05fff4cb8" member_type: VOTER last_known_addr { host: "127.11.34.254" port: 38675 } } peers { permanent_uuid: "abed6b5ab4604caa92637633d9f26bbf" member_type: VOTER last_known_addr { host: "127.11.34.253" port: 41205 } } peers { permanent_uuid: "215376717c434b229999d7279f7e12a6" member_type: VOTER last_known_addr { host: "127.11.34.252" port: 43017 } } }
20:07:20.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:20.168360 11718 sys_catalog.cc:458] T 00000000000000000000000000000000 P abed6b5ab4604caa92637633d9f26bbf [sys.catalog]: This master's current role is: FOLLOWER
20:07:20.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:20.168320 11705 sys_catalog.cc:458] T 00000000000000000000000000000000 P e65987b09c81467d8e563fb05fff4cb8 [sys.catalog]: This master's current role is: FOLLOWER
20:07:20.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:20.175772 11688 sys_catalog.cc:455] T 00000000000000000000000000000000 P 215376717c434b229999d7279f7e12a6 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "215376717c434b229999d7279f7e12a6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e65987b09c81467d8e563fb05fff4cb8" member_type: VOTER last_known_addr { host: "127.11.34.254" port: 38675 } } peers { permanent_uuid: "abed6b5ab4604caa92637633d9f26bbf" member_type: VOTER last_known_addr { host: "127.11.34.253" port: 41205 } } peers { permanent_uuid: "215376717c434b229999d7279f7e12a6" member_type: VOTER last_known_addr { host: "127.11.34.252" port: 43017 } } }
20:07:20.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:20.176473 11688 sys_catalog.cc:458] T 00000000000000000000000000000000 P 215376717c434b229999d7279f7e12a6 [sys.catalog]: This master's current role is: LEADER
20:07:20.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:20.184154 11736 sys_catalog.cc:455] T 00000000000000000000000000000000 P 215376717c434b229999d7279f7e12a6 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "215376717c434b229999d7279f7e12a6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e65987b09c81467d8e563fb05fff4cb8" member_type: VOTER last_known_addr { host: "127.11.34.254" port: 38675 } } peers { permanent_uuid: "abed6b5ab4604caa92637633d9f26bbf" member_type: VOTER last_known_addr { host: "127.11.34.253" port: 41205 } } peers { permanent_uuid: "215376717c434b229999d7279f7e12a6" member_type: VOTER last_known_addr { host: "127.11.34.252" port: 43017 } } }
20:07:20.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:20.187697 11736 sys_catalog.cc:458] T 00000000000000000000000000000000 P 215376717c434b229999d7279f7e12a6 [sys.catalog]: This master's current role is: LEADER
20:07:20.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:20.193147 11705 sys_catalog.cc:455] T 00000000000000000000000000000000 P e65987b09c81467d8e563fb05fff4cb8 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "215376717c434b229999d7279f7e12a6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e65987b09c81467d8e563fb05fff4cb8" member_type: VOTER last_known_addr { host: "127.11.34.254" port: 38675 } } peers { permanent_uuid: "abed6b5ab4604caa92637633d9f26bbf" member_type: VOTER last_known_addr { host: "127.11.34.253" port: 41205 } } peers { permanent_uuid: "215376717c434b229999d7279f7e12a6" member_type: VOTER last_known_addr { host: "127.11.34.252" port: 43017 } } }
20:07:20.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:20.193887 11705 sys_catalog.cc:458] T 00000000000000000000000000000000 P e65987b09c81467d8e563fb05fff4cb8 [sys.catalog]: This master's current role is: FOLLOWER
20:07:20.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:20.091435 11432 master_runner.cc:186] Error getting master registration for 127.11.34.253:41205: OK, instance_id { permanent_uuid: "abed6b5ab4604caa92637633d9f26bbf" instance_seqno: 1757102837502890 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
20:07:20.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:20.198729 11716 catalog_manager.cc:1349] Generated new cluster ID: 9cf977073414401480ad287067d54cc3
20:07:20.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:20.199029 11716 catalog_manager.cc:1497] Initializing Kudu internal certificate authority...
20:07:20.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:20.199386 11432 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:07:20.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:20.252698 11740 catalog_manager.cc:797] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
20:07:20.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:20.304091 11716 catalog_manager.cc:1372] Generated new certificate authority record
20:07:20.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:20.306193 11716 catalog_manager.cc:1506] Loading token signing keys...
20:07:20.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:20.329949 11716 catalog_manager.cc:5955] T 00000000000000000000000000000000 P 215376717c434b229999d7279f7e12a6: Generated new TSK 0
20:07:20.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:20.332077 11716 catalog_manager.cc:1516] Initializing in-progress tserver states...
20:07:20.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.11.34.193@KRBTEST.COM; defaulting to no policy
20:07:20.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:07:20.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.11.34.193@KRBTEST.COM" created.
20:07:20.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:07:20.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.11.34.193 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/krb5kdc/kudu.keytab.
20:07:20.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.11.34.193 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/krb5kdc/kudu.keytab.
20:07:20.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.11.34.193@KRBTEST.COM; defaulting to no policy
20:07:20.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:07:20.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.11.34.193@KRBTEST.COM" created.
20:07:20.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:07:20.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.11.34.193 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/krb5kdc/kudu.keytab.
20:07:20.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.11.34.193 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/krb5kdc/kudu.keytab.
20:07:20.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:20.450220 11403 external_mini_cluster.cc:1366] Running /tmp/dist-test-taske7evVR/build/tsan/bin/kudu
20:07:20.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taske7evVR/build/tsan/bin/kudu
20:07:20.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/ts-0/wal
20:07:20.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/ts-0/data
20:07:20.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:07:20.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:07:20.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:07:20.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:07:20.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:07:20.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:07:20.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/ts-0/logs
20:07:20.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/ts-0/data/info.pb
20:07:20.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:07:20.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:07:20.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:07:20.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:07:20.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:07:20.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:07:20.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:07:20.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:07:20.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
20:07:20.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:07:20.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.34.193:0
20:07:20.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.11.34.193
20:07:20.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.34.193
20:07:20.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:07:20.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.11.34.254:38675,127.11.34.253:41205,127.11.34.252:43017
20:07:20.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/krb5kdc/kudu.keytab
20:07:20.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.11.34.193
20:07:20.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:07:20.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:07:20.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:07:20.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
20:07:20.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.34.212:38019
20:07:20.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:07:20.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:07:20.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:07:20.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false with env {KRB5CCNAME=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:07:20.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:20.843508 11745 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:07:20.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:20.843977 11745 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:07:20.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:20.844641 11745 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:07:20.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:20.880882 11745 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:07:20.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:20.881878 11745 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.11.34.193
20:07:20.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:20.922559 11745 tablet_server_runner.cc:78] Tablet server non-default flags:
20:07:20.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:07:20.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.34.212:38019
20:07:20.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:07:20.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:07:20.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/ts-0/data
20:07:20.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/ts-0/wal
20:07:20.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/krb5kdc/kudu.keytab
20:07:20.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.11.34.193
20:07:20.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:07:20.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:07:20.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.34.193:0
20:07:20.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:07:20.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:07:20.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:07:20.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:07:20.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:07:20.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:07:20.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/ts-0/data/info.pb
20:07:20.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.34.193
20:07:20.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:07:20.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.11.34.254:38675,127.11.34.253:41205,127.11.34.252:43017
20:07:20.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:07:20.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:07:20.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:07:20.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:07:20.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:07:20.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.11.34.193
20:07:20.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/ts-0/logs
20:07:20.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:07:20.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:07:20.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:07:20.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
20:07:20.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:07:20.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision e2e116a9cb215058aad161061214d11c617cedac
20:07:20.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:07:20.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 05 Sep 2025 19:43:22 UTC on 5fd53c4cbb9d
20:07:20.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7917
20:07:20.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:07:20.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:20.924607 11745 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:07:20.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:20.926257 11745 file_cache.cc:492] Constructed file cache file cache with capacity 419430
20:07:20.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:20.942173 11754 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:07:21.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:21.111577 11720 catalog_manager.cc:1261] Loaded cluster ID: 9cf977073414401480ad287067d54cc3
20:07:21.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:21.111990 11720 catalog_manager.cc:1554] T 00000000000000000000000000000000 P e65987b09c81467d8e563fb05fff4cb8: loading cluster ID for follower catalog manager: success
20:07:21.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:21.129669 11720 catalog_manager.cc:1576] T 00000000000000000000000000000000 P e65987b09c81467d8e563fb05fff4cb8: acquiring CA information for follower catalog manager: success
20:07:21.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:21.143483 11720 catalog_manager.cc:1604] T 00000000000000000000000000000000 P e65987b09c81467d8e563fb05fff4cb8: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
20:07:21.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:21.270946 11740 catalog_manager.cc:1261] Loaded cluster ID: 9cf977073414401480ad287067d54cc3
20:07:21.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:21.271234 11740 catalog_manager.cc:1554] T 00000000000000000000000000000000 P abed6b5ab4604caa92637633d9f26bbf: loading cluster ID for follower catalog manager: success
20:07:21.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:21.307286 11740 catalog_manager.cc:1576] T 00000000000000000000000000000000 P abed6b5ab4604caa92637633d9f26bbf: acquiring CA information for follower catalog manager: success
20:07:21.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:21.317313 11740 catalog_manager.cc:1604] T 00000000000000000000000000000000 P abed6b5ab4604caa92637633d9f26bbf: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
20:07:23.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:20.944372 11755 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:07:23.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:23.222088 11757 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:07:23.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:23.221432 11756 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection timed out after 2276 milliseconds
20:07:23.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:22.344975 11753 debug-util.cc:398] Leaking SignalData structure 0x7b0800037b60 after lost signal to thread 11745
20:07:23.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:23.221710 11745 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 2.278s user 0.436s sys 0.749s
20:07:23.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:23.230779 11745 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 2.287s user 0.437s sys 0.749s
20:07:23.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:23.231117 11745 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:07:23.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:23.235587 11745 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:07:23.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:23.248108 11745 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:07:23.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:23.249536 11745 hybrid_clock.cc:648] HybridClock initialized: now 1757102843249462 us; error 93 us; skew 500 ppm
20:07:23.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:23.250671 11745 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:07:23.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 05 20:07:23 dist-test-slave-x5g2 krb5kdc[11415](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1757102843, etypes {rep=17 tkt=17 ses=17}, kudu/127.11.34.193@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:07:23.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:23.259174 11745 init.cc:377] Logged in from keytab as kudu/127.11.34.193@KRBTEST.COM (short username kudu)
20:07:23.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:23.338305 11745 webserver.cc:480] Webserver started at http://127.11.34.193:46549/ using document root <none> and password file <none>
20:07:23.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:23.339680 11745 fs_manager.cc:362] Metadata directory not provided
20:07:23.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:23.340009 11745 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:07:23.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:23.340580 11745 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
20:07:23.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:23.347222 11745 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/ts-0/data/instance:
20:07:23.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8544c1a7bcc94996b7d512785fb5655b"
20:07:23.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-05 20:07:23 on dist-test-slave-x5g2"
20:07:23.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:23.348802 11745 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/ts-0/wal/instance:
20:07:23.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8544c1a7bcc94996b7d512785fb5655b"
20:07:23.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-05 20:07:23 on dist-test-slave-x5g2"
20:07:23.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:23.365221 11745 fs_manager.cc:696] Time spent creating directory manager: real 0.016s user 0.009s sys 0.005s
20:07:23.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:23.378436 11772 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:07:23.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:23.379902 11745 fs_manager.cc:730] Time spent opening block manager: real 0.010s user 0.002s sys 0.006s
20:07:23.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:23.380347 11745 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/ts-0/data,/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/ts-0/wal
20:07:23.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8544c1a7bcc94996b7d512785fb5655b"
20:07:23.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-05 20:07:23 on dist-test-slave-x5g2"
20:07:23.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:23.380800 11745 fs_report.cc:389] FS layout report
20:07:23.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:07:23.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/ts-0/wal
20:07:23.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/ts-0/wal
20:07:23.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/ts-0/data/data
20:07:23.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:07:23.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:07:23.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:07:23.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:07:23.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:07:23.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:07:23.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:07:23.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:07:23.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:07:23.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:07:23.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:07:23.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:23.503685 11745 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:07:23.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:23.512045 11745 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:07:23.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:23.512624 11745 kserver.cc:163] Server-wide thread pool size limit: 3276
20:07:23.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:23.521898 11745 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
20:07:23.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:23.534660 11745 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
20:07:23.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:23.534983 11745 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
20:07:23.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:23.535276 11745 ts_tablet_manager.cc:610] Registered 0 tablets
20:07:23.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:23.535498 11745 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
20:07:23.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:23.844573 11745 rpc_server.cc:307] RPC server started. Bound to: 127.11.34.193:41147
20:07:23.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:23.844720 11886 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.11.34.193:41147 every 8 connection(s)
20:07:23.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:23.848345 11745 server_base.cc:1179] Dumped server information to /tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/ts-0/data/info.pb
20:07:23.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:23.857587 11403 external_mini_cluster.cc:1428] Started /tmp/dist-test-taske7evVR/build/tsan/bin/kudu as pid 11745
20:07:23.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:23.858243 11403 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/ts-0/wal/instance
20:07:23.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 05 20:07:23 dist-test-slave-x5g2 krb5kdc[11415](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1757102843, etypes {rep=17 tkt=17 ses=17}, kudu/127.11.34.193@KRBTEST.COM for kudu/127.11.34.252@KRBTEST.COM
20:07:23.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:23.936124 11887 heartbeater.cc:344] Connected to a master server at 127.11.34.252:43017
20:07:23.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:23.936620 11887 heartbeater.cc:461] Registering TS with master...
20:07:23.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:23.938081 11887 heartbeater.cc:507] Master 127.11.34.252:43017 requested a full tablet report, sending...
20:07:23.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:23.941569 11643 ts_manager.cc:194] Registered new tserver with Master: 8544c1a7bcc94996b7d512785fb5655b (127.11.34.193:41147)
20:07:23.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:23.944298 11643 master_service.cc:496] Signed X509 certificate for tserver {username='kudu', principal='kudu/127.11.34.193@KRBTEST.COM'} at 127.11.34.193:46799
20:07:23.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.11.34.194@KRBTEST.COM; defaulting to no policy
20:07:23.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:07:23.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.11.34.194@KRBTEST.COM" created.
20:07:23.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 05 20:07:23 dist-test-slave-x5g2 krb5kdc[11415](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1757102843, etypes {rep=17 tkt=17 ses=17}, kudu/127.11.34.193@KRBTEST.COM for kudu/127.11.34.254@KRBTEST.COM
20:07:23.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 05 20:07:23 dist-test-slave-x5g2 krb5kdc[11415](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1757102843, etypes {rep=17 tkt=17 ses=17}, kudu/127.11.34.193@KRBTEST.COM for kudu/127.11.34.253@KRBTEST.COM
20:07:23.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:23.991113 11888 heartbeater.cc:344] Connected to a master server at 127.11.34.253:41205
20:07:23.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:23.991533 11888 heartbeater.cc:461] Registering TS with master...
20:07:23.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:23.992417 11888 heartbeater.cc:507] Master 127.11.34.253:41205 requested a full tablet report, sending...
20:07:23.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:23.995532 11891 heartbeater.cc:344] Connected to a master server at 127.11.34.254:38675
20:07:23.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:23.995944 11891 heartbeater.cc:461] Registering TS with master...
20:07:23.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:23.996021 11555 ts_manager.cc:194] Registered new tserver with Master: 8544c1a7bcc94996b7d512785fb5655b (127.11.34.193:41147)
20:07:23.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:23.996623 11891 heartbeater.cc:507] Master 127.11.34.254:38675 requested a full tablet report, sending...
20:07:24.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:24.001430 11467 ts_manager.cc:194] Registered new tserver with Master: 8544c1a7bcc94996b7d512785fb5655b (127.11.34.193:41147)
20:07:24.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:07:24.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.11.34.194 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/krb5kdc/kudu.keytab.
20:07:24.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.11.34.194 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/krb5kdc/kudu.keytab.
20:07:24.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:24.017264 11403 mini_kdc.cc:279] Time spent creating service keytab for kudu/127.11.34.194: real 0.124s user 0.001s sys 0.013s
20:07:24.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.11.34.194@KRBTEST.COM; defaulting to no policy
20:07:24.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:07:24.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.11.34.194@KRBTEST.COM" created.
20:07:24.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:07:24.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.11.34.194 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/krb5kdc/kudu.keytab.
20:07:24.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.11.34.194 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/krb5kdc/kudu.keytab.
20:07:24.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:24.077986 11403 external_mini_cluster.cc:1366] Running /tmp/dist-test-taske7evVR/build/tsan/bin/kudu
20:07:24.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taske7evVR/build/tsan/bin/kudu
20:07:24.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/ts-1/wal
20:07:24.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/ts-1/data
20:07:24.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:07:24.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:07:24.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:07:24.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:07:24.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:07:24.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:07:24.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/ts-1/logs
20:07:24.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/ts-1/data/info.pb
20:07:24.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:07:24.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:07:24.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:07:24.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:07:24.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:07:24.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:07:24.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:07:24.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:07:24.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
20:07:24.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:07:24.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.34.194:0
20:07:24.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.11.34.194
20:07:24.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.34.194
20:07:24.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:07:24.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.11.34.254:38675,127.11.34.253:41205,127.11.34.252:43017
20:07:24.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/krb5kdc/kudu.keytab
20:07:24.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.11.34.194
20:07:24.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:07:24.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:07:24.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:07:24.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
20:07:24.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.34.212:38019
20:07:24.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:07:24.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:07:24.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:07:24.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false with env {KRB5CCNAME=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:07:24.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:24.576953 11900 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:07:24.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:24.577962 11900 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:07:24.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:24.579166 11900 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:07:24.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:24.639214 11900 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:07:24.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:24.641166 11900 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.11.34.194
20:07:24.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:24.711028 11900 tablet_server_runner.cc:78] Tablet server non-default flags:
20:07:24.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:07:24.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.34.212:38019
20:07:24.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:07:24.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:07:24.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/ts-1/data
20:07:24.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/ts-1/wal
20:07:24.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/krb5kdc/kudu.keytab
20:07:24.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.11.34.194
20:07:24.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:07:24.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:07:24.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.34.194:0
20:07:24.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:07:24.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:07:24.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:07:24.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:07:24.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:07:24.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:07:24.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/ts-1/data/info.pb
20:07:24.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.34.194
20:07:24.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:07:24.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.11.34.254:38675,127.11.34.253:41205,127.11.34.252:43017
20:07:24.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:07:24.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:07:24.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:07:24.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:07:24.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:07:24.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.11.34.194
20:07:24.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/ts-1/logs
20:07:24.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:07:24.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:07:24.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:07:24.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
20:07:24.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:07:24.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision e2e116a9cb215058aad161061214d11c617cedac
20:07:24.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:07:24.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 05 Sep 2025 19:43:22 UTC on 5fd53c4cbb9d
20:07:24.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7917
20:07:24.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:07:24.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:24.713050 11900 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:07:24.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:24.714548 11900 file_cache.cc:492] Constructed file cache file cache with capacity 419430
20:07:24.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:24.732059 11909 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:07:24.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:24.952915 11887 heartbeater.cc:499] Master 127.11.34.252:43017 was elected leader, sending a full tablet report...
20:07:24.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 05 20:07:24 dist-test-slave-x5g2 krb5kdc[11415](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1757102844, etypes {rep=17 tkt=17 ses=17}, kudu/127.11.34.254@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:07:24.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:24.985471 11446 init.cc:295] Successfully reacquired a new kerberos TGT
20:07:27.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:24.736150 11910 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:07:27.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:27.113951 11900 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 2.378s user 0.584s sys 0.958s
20:07:27.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:27.115120 11900 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 2.379s user 0.584s sys 0.959s
20:07:27.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:26.134546 11908 debug-util.cc:398] Leaking SignalData structure 0x7b0800037b60 after lost signal to thread 11900
20:07:27.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:27.117216 11912 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:07:27.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:27.119700 11911 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2379 milliseconds
20:07:27.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:27.119766 11900 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:07:27.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:27.121022 11900 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:07:27.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:27.123325 11900 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:07:27.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:27.124707 11900 hybrid_clock.cc:648] HybridClock initialized: now 1757102847124652 us; error 56 us; skew 500 ppm
20:07:27.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:27.125705 11900 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:07:27.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 05 20:07:27 dist-test-slave-x5g2 krb5kdc[11415](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1757102847, etypes {rep=17 tkt=17 ses=17}, kudu/127.11.34.194@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:07:27.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:27.132158 11900 init.cc:377] Logged in from keytab as kudu/127.11.34.194@KRBTEST.COM (short username kudu)
20:07:27.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:27.137280 11900 webserver.cc:480] Webserver started at http://127.11.34.194:38631/ using document root <none> and password file <none>
20:07:27.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:27.138358 11900 fs_manager.cc:362] Metadata directory not provided
20:07:27.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:27.138604 11900 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:07:27.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:27.139029 11900 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
20:07:27.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:27.144562 11900 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/ts-1/data/instance:
20:07:27.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4d1c197356254990aac35983eb968fe0"
20:07:27.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-05 20:07:27 on dist-test-slave-x5g2"
20:07:27.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:27.145722 11900 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/ts-1/wal/instance:
20:07:27.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4d1c197356254990aac35983eb968fe0"
20:07:27.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-05 20:07:27 on dist-test-slave-x5g2"
20:07:27.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:27.153584 11900 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.006s sys 0.001s
20:07:27.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:27.159404 11924 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:07:27.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:27.160544 11900 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.003s
20:07:27.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:27.160845 11900 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/ts-1/data,/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/ts-1/wal
20:07:27.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4d1c197356254990aac35983eb968fe0"
20:07:27.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-05 20:07:27 on dist-test-slave-x5g2"
20:07:27.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:27.161152 11900 fs_report.cc:389] FS layout report
20:07:27.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:07:27.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/ts-1/wal
20:07:27.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/ts-1/wal
20:07:27.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/ts-1/data/data
20:07:27.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:07:27.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:07:27.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:07:27.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:07:27.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:07:27.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:07:27.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:07:27.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:07:27.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:07:27.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:07:27.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:07:27.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:27.227870 11900 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:07:27.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:27.239259 11900 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:07:27.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:27.240192 11900 kserver.cc:163] Server-wide thread pool size limit: 3276
20:07:27.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:27.242872 11900 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
20:07:27.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:27.247826 11900 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
20:07:27.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:27.248116 11900 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
20:07:27.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:27.248399 11900 ts_tablet_manager.cc:610] Registered 0 tablets
20:07:27.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:27.248608 11900 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
20:07:27.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 05 20:07:27 dist-test-slave-x5g2 krb5kdc[11415](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1757102847, etypes {rep=17 tkt=17 ses=17}, kudu/127.11.34.253@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:07:27.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:27.351496 11534 init.cc:295] Successfully reacquired a new kerberos TGT
20:07:27.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:27.418653 11900 rpc_server.cc:307] RPC server started. Bound to: 127.11.34.194:34845
20:07:27.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:27.418771 12036 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.11.34.194:34845 every 8 connection(s)
20:07:27.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:27.421270 11900 server_base.cc:1179] Dumped server information to /tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/ts-1/data/info.pb
20:07:27.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:27.432343 11403 external_mini_cluster.cc:1428] Started /tmp/dist-test-taske7evVR/build/tsan/bin/kudu as pid 11900
20:07:27.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:27.432895 11403 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/ts-1/wal/instance
20:07:27.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 05 20:07:27 dist-test-slave-x5g2 krb5kdc[11415](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1757102847, etypes {rep=17 tkt=17 ses=17}, kudu/127.11.34.194@KRBTEST.COM for kudu/127.11.34.253@KRBTEST.COM
20:07:27.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 05 20:07:27 dist-test-slave-x5g2 krb5kdc[11415](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1757102847, etypes {rep=17 tkt=17 ses=17}, kudu/127.11.34.194@KRBTEST.COM for kudu/127.11.34.252@KRBTEST.COM
20:07:27.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.11.34.195@KRBTEST.COM; defaulting to no policy
20:07:27.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:07:27.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.11.34.195@KRBTEST.COM" created.
20:07:27.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 05 20:07:27 dist-test-slave-x5g2 krb5kdc[11415](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1757102847, etypes {rep=17 tkt=17 ses=17}, kudu/127.11.34.194@KRBTEST.COM for kudu/127.11.34.254@KRBTEST.COM
20:07:27.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:27.501720 12038 heartbeater.cc:344] Connected to a master server at 127.11.34.253:41205
20:07:27.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:27.502278 12038 heartbeater.cc:461] Registering TS with master...
20:07:27.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:27.503762 12038 heartbeater.cc:507] Master 127.11.34.253:41205 requested a full tablet report, sending...
20:07:27.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:27.508051 11555 ts_manager.cc:194] Registered new tserver with Master: 4d1c197356254990aac35983eb968fe0 (127.11.34.194:34845)
20:07:27.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:27.511086 12037 heartbeater.cc:344] Connected to a master server at 127.11.34.252:43017
20:07:27.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:27.511447 12037 heartbeater.cc:461] Registering TS with master...
20:07:27.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:27.512539 12037 heartbeater.cc:507] Master 127.11.34.252:43017 requested a full tablet report, sending...
20:07:27.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:27.514951 11643 ts_manager.cc:194] Registered new tserver with Master: 4d1c197356254990aac35983eb968fe0 (127.11.34.194:34845)
20:07:27.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:07:27.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.11.34.195 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/krb5kdc/kudu.keytab.
20:07:27.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.11.34.195 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/krb5kdc/kudu.keytab.
20:07:27.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:27.516755 11643 master_service.cc:496] Signed X509 certificate for tserver {username='kudu', principal='kudu/127.11.34.194@KRBTEST.COM'} at 127.11.34.194:48685
20:07:27.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:27.532625 12039 heartbeater.cc:344] Connected to a master server at 127.11.34.254:38675
20:07:27.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:27.533146 12039 heartbeater.cc:461] Registering TS with master...
20:07:27.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:27.533974 12039 heartbeater.cc:507] Master 127.11.34.254:38675 requested a full tablet report, sending...
20:07:27.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:27.536538 11467 ts_manager.cc:194] Registered new tserver with Master: 4d1c197356254990aac35983eb968fe0 (127.11.34.194:34845)
20:07:27.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.11.34.195@KRBTEST.COM; defaulting to no policy
20:07:27.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:07:27.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.11.34.195@KRBTEST.COM" created.
20:07:27.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:07:27.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.11.34.195 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/krb5kdc/kudu.keytab.
20:07:27.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.11.34.195 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/krb5kdc/kudu.keytab.
20:07:27.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:27.584241 11403 external_mini_cluster.cc:1366] Running /tmp/dist-test-taske7evVR/build/tsan/bin/kudu
20:07:27.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taske7evVR/build/tsan/bin/kudu
20:07:27.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/ts-2/wal
20:07:27.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/ts-2/data
20:07:27.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:07:27.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:07:27.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:07:27.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:07:27.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:07:27.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:07:27.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/ts-2/logs
20:07:27.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/ts-2/data/info.pb
20:07:27.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:07:27.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:07:27.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:07:27.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:07:27.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:07:27.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:07:27.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:07:27.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:07:27.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
20:07:27.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:07:27.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.34.195:0
20:07:27.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.11.34.195
20:07:27.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.34.195
20:07:27.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:07:27.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.11.34.254:38675,127.11.34.253:41205,127.11.34.252:43017
20:07:27.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/krb5kdc/kudu.keytab
20:07:27.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.11.34.195
20:07:27.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:07:27.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:07:27.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:07:27.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
20:07:27.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.34.212:38019
20:07:27.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:07:27.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:07:27.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:07:27.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false with env {KRB5CCNAME=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:07:28.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:28.032833 12051 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:07:28.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:28.033376 12051 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:07:28.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:28.034164 12051 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:07:28.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:28.067548 12051 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:07:28.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:28.068536 12051 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.11.34.195
20:07:28.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:28.107316 12051 tablet_server_runner.cc:78] Tablet server non-default flags:
20:07:28.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:07:28.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.34.212:38019
20:07:28.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:07:28.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:07:28.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/ts-2/data
20:07:28.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/ts-2/wal
20:07:28.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/krb5kdc/kudu.keytab
20:07:28.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.11.34.195
20:07:28.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:07:28.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:07:28.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.34.195:0
20:07:28.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:07:28.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:07:28.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:07:28.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:07:28.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:07:28.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:07:28.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/ts-2/data/info.pb
20:07:28.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.34.195
20:07:28.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:07:28.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.11.34.254:38675,127.11.34.253:41205,127.11.34.252:43017
20:07:28.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:07:28.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:07:28.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:07:28.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:07:28.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:07:28.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.11.34.195
20:07:28.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/ts-2/logs
20:07:28.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:07:28.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:07:28.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:07:28.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
20:07:28.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:07:28.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision e2e116a9cb215058aad161061214d11c617cedac
20:07:28.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:07:28.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 05 Sep 2025 19:43:22 UTC on 5fd53c4cbb9d
20:07:28.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7917
20:07:28.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:07:28.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:28.109090 12051 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:07:28.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:28.110260 12051 file_cache.cc:492] Constructed file cache file cache with capacity 419430
20:07:28.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:28.123785 12058 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:07:28.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:28.521773 12037 heartbeater.cc:499] Master 127.11.34.252:43017 was elected leader, sending a full tablet report...
20:07:29.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 05 20:07:29 dist-test-slave-x5g2 krb5kdc[11415](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1757102849, etypes {rep=17 tkt=17 ses=17}, kudu/127.11.34.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:07:29.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:29.396862 11622 init.cc:295] Successfully reacquired a new kerberos TGT
20:07:30.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:29.534974 12057 debug-util.cc:398] Leaking SignalData structure 0x7b0800037b60 after lost signal to thread 12051
20:07:30.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:30.150279 12057 kernel_stack_watchdog.cc:198] Thread 12051 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 408ms:
20:07:30.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
20:07:30.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
20:07:30.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
20:07:30.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
20:07:30.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:28.124434 12059 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:07:30.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:30.151192 12051 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 2.027s user 0.737s sys 1.174s
20:07:30.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:30.151750 12051 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 2.028s user 0.737s sys 1.175s
20:07:30.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:30.153110 12062 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:07:30.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:30.157421 12060 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2026 milliseconds
20:07:30.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:30.157537 12051 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:07:30.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:30.158926 12051 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:07:30.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:30.161821 12051 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:07:30.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:30.163237 12051 hybrid_clock.cc:648] HybridClock initialized: now 1757102850163185 us; error 54 us; skew 500 ppm
20:07:30.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:30.164304 12051 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:07:30.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 05 20:07:30 dist-test-slave-x5g2 krb5kdc[11415](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1757102850, etypes {rep=17 tkt=17 ses=17}, kudu/127.11.34.195@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:07:30.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:30.172122 12051 init.cc:377] Logged in from keytab as kudu/127.11.34.195@KRBTEST.COM (short username kudu)
20:07:30.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:30.178277 12051 webserver.cc:480] Webserver started at http://127.11.34.195:39345/ using document root <none> and password file <none>
20:07:30.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:30.179188 12051 fs_manager.cc:362] Metadata directory not provided
20:07:30.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:30.179385 12051 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:07:30.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:30.180538 12051 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
20:07:30.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:30.187330 12051 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/ts-2/data/instance:
20:07:30.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d0af648559814b0d9e9af1b19e0eccf1"
20:07:30.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-05 20:07:30 on dist-test-slave-x5g2"
20:07:30.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:30.189060 12051 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/ts-2/wal/instance:
20:07:30.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d0af648559814b0d9e9af1b19e0eccf1"
20:07:30.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-05 20:07:30 on dist-test-slave-x5g2"
20:07:30.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:30.206192 12051 fs_manager.cc:696] Time spent creating directory manager: real 0.016s user 0.009s sys 0.005s
20:07:30.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:30.214540 12071 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:07:30.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:30.216068 12051 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.001s sys 0.003s
20:07:30.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:30.216517 12051 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/ts-2/data,/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/ts-2/wal
20:07:30.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d0af648559814b0d9e9af1b19e0eccf1"
20:07:30.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-05 20:07:30 on dist-test-slave-x5g2"
20:07:30.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:30.216991 12051 fs_report.cc:389] FS layout report
20:07:30.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:07:30.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/ts-2/wal
20:07:30.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/ts-2/wal
20:07:30.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/ts-2/data/data
20:07:30.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:07:30.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:07:30.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:07:30.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:07:30.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:07:30.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:07:30.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:07:30.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:07:30.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:07:30.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:07:30.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:07:30.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:30.301360 12051 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:07:30.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:30.309737 12051 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:07:30.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:30.310333 12051 kserver.cc:163] Server-wide thread pool size limit: 3276
20:07:30.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:30.314105 12051 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
20:07:30.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:30.320416 12051 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
20:07:30.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:30.320739 12051 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
20:07:30.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:30.321020 12051 ts_tablet_manager.cc:610] Registered 0 tablets
20:07:30.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:30.321215 12051 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
20:07:30.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:30.573354 12051 rpc_server.cc:307] RPC server started. Bound to: 127.11.34.195:33453
20:07:30.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:30.573559 12183 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.11.34.195:33453 every 8 connection(s)
20:07:30.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:30.576231 12051 server_base.cc:1179] Dumped server information to /tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/ts-2/data/info.pb
20:07:30.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:30.579974 11403 external_mini_cluster.cc:1428] Started /tmp/dist-test-taske7evVR/build/tsan/bin/kudu as pid 12051
20:07:30.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:30.580591 11403 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/ts-2/wal/instance
20:07:30.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 05 20:07:30 dist-test-slave-x5g2 krb5kdc[11415](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1757102850, etypes {rep=17 tkt=17 ses=17}, kudu/127.11.34.195@KRBTEST.COM for kudu/127.11.34.253@KRBTEST.COM
20:07:30.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 05 20:07:30 dist-test-slave-x5g2 krb5kdc[11415](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1757102850, etypes {rep=17 tkt=17 ses=17}, kudu/127.11.34.195@KRBTEST.COM for kudu/127.11.34.252@KRBTEST.COM
20:07:30.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:30.691326 12185 heartbeater.cc:344] Connected to a master server at 127.11.34.253:41205
20:07:30.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:30.692116 12185 heartbeater.cc:461] Registering TS with master...
20:07:30.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:30.694727 12185 heartbeater.cc:507] Master 127.11.34.253:41205 requested a full tablet report, sending...
20:07:30.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:30.698513 11555 ts_manager.cc:194] Registered new tserver with Master: d0af648559814b0d9e9af1b19e0eccf1 (127.11.34.195:33453)
20:07:30.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 05 20:07:30 dist-test-slave-x5g2 krb5kdc[11415](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1757102850, etypes {rep=17 tkt=17 ses=17}, kudu/127.11.34.195@KRBTEST.COM for kudu/127.11.34.254@KRBTEST.COM
20:07:30.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:30.715363 12184 heartbeater.cc:344] Connected to a master server at 127.11.34.252:43017
20:07:30.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:30.715806 12184 heartbeater.cc:461] Registering TS with master...
20:07:30.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:30.716739 12184 heartbeater.cc:507] Master 127.11.34.252:43017 requested a full tablet report, sending...
20:07:30.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:30.720444 11643 ts_manager.cc:194] Registered new tserver with Master: d0af648559814b0d9e9af1b19e0eccf1 (127.11.34.195:33453)
20:07:30.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:30.722311 11643 master_service.cc:496] Signed X509 certificate for tserver {username='kudu', principal='kudu/127.11.34.195@KRBTEST.COM'} at 127.11.34.195:32943
20:07:30.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:30.729110 12188 heartbeater.cc:344] Connected to a master server at 127.11.34.254:38675
20:07:30.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:30.729562 12188 heartbeater.cc:461] Registering TS with master...
20:07:30.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:30.730221 12188 heartbeater.cc:507] Master 127.11.34.254:38675 requested a full tablet report, sending...
20:07:30.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:30.732403 11467 ts_manager.cc:194] Registered new tserver with Master: d0af648559814b0d9e9af1b19e0eccf1 (127.11.34.195:33453)
20:07:30.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:30.745355 11403 external_mini_cluster.cc:949] 3 TS(s) registered with all masters
20:07:30.757 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:07:30.758 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
20:07:30.769 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
20:07:30.770 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
20:07:30.773 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.11.34.254"
port: 38675
}
bound_http_address {
host: "127.11.34.254"
port: 44601
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.11.34.253"
port: 41205
}
bound_http_address {
host: "127.11.34.253"
port: 33625
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.11.34.252"
port: 43017
}
bound_http_address {
host: "127.11.34.252"
port: 38995
}
}
}
20:07:30.774 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
20:07:30.775 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.11.34.193"
port: 41147
}
bound_http_address {
host: "127.11.34.193"
port: 46549
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.11.34.194"
port: 34845
}
bound_http_address {
host: "127.11.34.194"
port: 38631
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.11.34.195"
port: 33453
}
bound_http_address {
host: "127.11.34.195"
port: 39345
}
}
}
20:07:30.776 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
20:07:30.777 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
20:07:30.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 05 20:07:30 dist-test-slave-x5g2 krb5kdc[11415](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1757102850, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:07:30.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
20:07:30.810 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:07:30.812 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/krb5kdc/krb5cc
20:07:30.818 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
20:07:30.819 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/krb5kdc/krb5cc
20:07:30.820 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
20:07:30.824 [DEBUG - Time-limited test] (SecurityContext.java:158) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
20:07:30.826 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
20:07:30.826 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
20:07:30.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 05 20:07:30 dist-test-slave-x5g2 krb5kdc[11415](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1757102850, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:07:30.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
20:07:30.864 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:07:30.864 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/krb5kdc/krb5cc
20:07:30.865 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
20:07:30.866 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.11.34.252:43017
20:07:30.867 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
20:07:30.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:30.868371 11403 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taske7evVR/build/tsan/bin/kudu with pid 11603
20:07:30.890 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:07:30.890 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.11.34.254:38675
20:07:30.891 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
20:07:30.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:30.891310 11403 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taske7evVR/build/tsan/bin/kudu with pid 11432
20:07:30.921 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:07:30.921 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.11.34.253:41205
20:07:30.922 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
20:07:30.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:30.922377 11403 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taske7evVR/build/tsan/bin/kudu with pid 11511
20:07:30.952 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:07:30.952 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.11.34.252:43017
20:07:30.956 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
20:07:30.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:30.959036 11403 external_mini_cluster.cc:1366] Running /tmp/dist-test-taske7evVR/build/tsan/bin/kudu
20:07:30.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taske7evVR/build/tsan/bin/kudu
20:07:30.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/master-2/wal
20:07:30.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/master-2/data
20:07:30.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:07:30.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:07:30.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:07:30.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:07:30.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:07:30.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:07:30.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/master-2/logs
20:07:30.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/master-2/data/info.pb
20:07:30.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:07:30.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:07:30.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:07:30.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:07:30.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:07:30.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:07:30.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:07:30.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:07:30.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:07:30.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:07:30.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:07:30.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:07:30.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.34.252:43017
20:07:30.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.34.252
20:07:30.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38995
20:07:30.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/krb5kdc/kudu.keytab
20:07:30.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.11.34.252
20:07:30.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:07:30.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:07:30.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:07:30.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
20:07:30.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.34.212:38019
20:07:30.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:07:30.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:07:30.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:07:30.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:07:30.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.11.34.254:38675,127.11.34.253:41205,127.11.34.252:43017
20:07:30.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:07:30.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34155/jwks.json
20:07:30.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
20:07:30.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:07:30.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:30.990010 11887 heartbeater.cc:646] Failed to heartbeat to 127.11.34.252:43017 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.11.34.252:43017: connect: Connection refused (error 111)
20:07:31.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:31.277266 12201 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:07:31.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:31.278046 12201 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:07:31.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:31.278802 12201 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:07:31.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:31.312378 12201 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
20:07:31.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:31.312682 12201 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
20:07:31.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:31.313136 12201 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:07:31.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:31.313473 12201 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
20:07:31.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:31.313786 12201 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
20:07:31.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:31.314105 12201 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
20:07:31.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:31.314322 12201 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34155/jwks.json
20:07:31.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:31.372769 12201 master_runner.cc:386] Master server non-default flags:
20:07:31.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:07:31.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.34.212:38019
20:07:31.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:07:31.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:07:31.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/master-2/data
20:07:31.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/master-2/wal
20:07:31.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
20:07:31.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:07:31.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.11.34.254:38675,127.11.34.253:41205,127.11.34.252:43017
20:07:31.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/krb5kdc/kudu.keytab
20:07:31.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.11.34.252
20:07:31.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:07:31.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:07:31.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:07:31.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.34.252:43017
20:07:31.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:07:31.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:07:31.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:07:31.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34155/jwks.json
20:07:31.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:07:31.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:07:31.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:07:31.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:07:31.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:07:31.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/master-2/data/info.pb
20:07:31.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.34.252
20:07:31.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38995
20:07:31.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:07:31.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:07:31.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:07:31.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:07:31.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:07:31.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/master-2/logs
20:07:31.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:07:31.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:07:31.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:07:31.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:07:31.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:07:31.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision e2e116a9cb215058aad161061214d11c617cedac
20:07:31.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:07:31.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 05 Sep 2025 19:43:22 UTC on 5fd53c4cbb9d
20:07:31.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7917
20:07:31.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:07:31.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:31.374763 12201 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:07:31.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:31.376156 12201 file_cache.cc:492] Constructed file cache file cache with capacity 419430
20:07:31.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:31.391418 12208 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:07:31.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:31.553457 12038 heartbeater.cc:646] Failed to heartbeat to 127.11.34.253:41205 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.11.34.253:41205: connect: Connection refused (error 111)
20:07:31.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:31.723392 12185 heartbeater.cc:646] Failed to heartbeat to 127.11.34.253:41205 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.11.34.253:41205: connect: Connection refused (error 111)
20:07:31.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:31.727141 12184 heartbeater.cc:499] Master 127.11.34.252:43017 was elected leader, sending a full tablet report...
20:07:32.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:32.421785 11387 debug-util.cc:398] Leaking SignalData structure 0x7b08000ac6c0 after lost signal to thread 11262
20:07:32.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:32.423419 11387 debug-util.cc:398] Leaking SignalData structure 0x7b08000b5b00 after lost signal to thread 11390
20:07:33.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 05 20:07:33 dist-test-slave-x5g2 krb5kdc[11415](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1757102853, etypes {rep=17 tkt=17 ses=17}, kudu/127.11.34.193@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:07:33.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:33.285894 11768 init.cc:295] Successfully reacquired a new kerberos TGT
20:07:33.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:32.793777 12207 debug-util.cc:398] Leaking SignalData structure 0x7b0800037200 after lost signal to thread 12201
20:07:33.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:33.298540 12207 kernel_stack_watchdog.cc:198] Thread 12201 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 396ms:
20:07:33.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
20:07:33.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
20:07:33.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
20:07:33.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
20:07:33.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:31.395360 12209 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:07:33.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:33.306936 12201 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.912s user 0.529s sys 1.063s
20:07:33.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:33.307351 12201 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.913s user 0.529s sys 1.063s
20:07:33.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:33.308239 12211 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:07:33.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:33.309196 12210 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1912 milliseconds
20:07:33.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:33.309257 12201 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:07:33.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:33.310827 12201 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:07:33.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:33.320073 12201 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:07:33.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:33.321602 12201 hybrid_clock.cc:648] HybridClock initialized: now 1757102853321513 us; error 89 us; skew 500 ppm
20:07:33.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:33.322786 12201 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:07:33.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 05 20:07:33 dist-test-slave-x5g2 krb5kdc[11415](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1757102853, etypes {rep=17 tkt=17 ses=17}, kudu/127.11.34.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:07:33.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:33.331012 12201 init.cc:377] Logged in from keytab as kudu/127.11.34.252@KRBTEST.COM (short username kudu)
20:07:33.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:33.336777 12201 webserver.cc:480] Webserver started at http://127.11.34.252:38995/ using document root <none> and password file <none>
20:07:33.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:33.338033 12201 fs_manager.cc:362] Metadata directory not provided
20:07:33.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:33.338307 12201 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:07:33.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:33.347488 12201 fs_manager.cc:714] Time spent opening directory manager: real 0.006s user 0.000s sys 0.004s
20:07:33.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:33.352504 12228 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:07:33.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:33.355170 12201 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.000s sys 0.002s
20:07:33.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:33.355504 12201 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/master-2/data,/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/master-2/wal
20:07:33.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "215376717c434b229999d7279f7e12a6"
20:07:33.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-05 20:07:19 on dist-test-slave-x5g2"
20:07:33.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:33.357614 12201 fs_report.cc:389] FS layout report
20:07:33.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:07:33.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/master-2/wal
20:07:33.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/master-2/wal
20:07:33.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/master-2/data/data
20:07:33.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:07:33.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:07:33.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:07:33.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:07:33.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:07:33.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:07:33.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:07:33.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:07:33.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:07:33.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:07:33.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:07:33.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:33.428390 12201 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:07:33.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:33.442353 12201 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:07:33.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:33.442945 12201 kserver.cc:163] Server-wide thread pool size limit: 3276
20:07:33.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:33.554239 12201 rpc_server.cc:307] RPC server started. Bound to: 127.11.34.252:43017
20:07:33.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:33.554291 12279 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.11.34.252:43017 every 8 connection(s)
20:07:33.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:33.558454 12201 server_base.cc:1179] Dumped server information to /tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/master-2/data/info.pb
20:07:33.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:33.562342 11403 external_mini_cluster.cc:1428] Started /tmp/dist-test-taske7evVR/build/tsan/bin/kudu as pid 12201
20:07:33.563 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:07:33.563 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.11.34.254:38675
20:07:33.563 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
20:07:33.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:33.565716 11403 external_mini_cluster.cc:1366] Running /tmp/dist-test-taske7evVR/build/tsan/bin/kudu
20:07:33.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taske7evVR/build/tsan/bin/kudu
20:07:33.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/master-0/wal
20:07:33.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/master-0/data
20:07:33.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:07:33.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:07:33.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:07:33.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:07:33.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:07:33.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:07:33.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/master-0/logs
20:07:33.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/master-0/data/info.pb
20:07:33.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:07:33.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:07:33.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:07:33.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:07:33.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:07:33.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:07:33.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:07:33.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:07:33.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:07:33.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:07:33.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:07:33.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:07:33.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.34.254:38675
20:07:33.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.34.254
20:07:33.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44601
20:07:33.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/krb5kdc/kudu.keytab
20:07:33.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.11.34.254
20:07:33.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:07:33.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:07:33.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:07:33.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
20:07:33.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.34.212:38019
20:07:33.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:07:33.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:07:33.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:07:33.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:07:33.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.11.34.254:38675,127.11.34.253:41205,127.11.34.252:43017
20:07:33.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:07:33.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34155/jwks.json
20:07:33.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
20:07:33.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:07:33.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:33.581741 12280 sys_catalog.cc:263] Verifying existing consensus state
20:07:33.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:33.596992 12280 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 215376717c434b229999d7279f7e12a6: Bootstrap starting.
20:07:33.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:33.672412 12037 heartbeater.cc:344] Connected to a master server at 127.11.34.252:43017
20:07:33.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:33.696897 12280 log.cc:826] T 00000000000000000000000000000000 P 215376717c434b229999d7279f7e12a6: Log is configured to *not* fsync() on all Append() calls
20:07:33.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:33.719229 12280 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 215376717c434b229999d7279f7e12a6: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
20:07:33.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:33.720381 12280 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 215376717c434b229999d7279f7e12a6: Bootstrap complete.
20:07:33.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:33.752578 12280 raft_consensus.cc:357] T 00000000000000000000000000000000 P 215376717c434b229999d7279f7e12a6 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e65987b09c81467d8e563fb05fff4cb8" member_type: VOTER last_known_addr { host: "127.11.34.254" port: 38675 } } peers { permanent_uuid: "abed6b5ab4604caa92637633d9f26bbf" member_type: VOTER last_known_addr { host: "127.11.34.253" port: 41205 } } peers { permanent_uuid: "215376717c434b229999d7279f7e12a6" member_type: VOTER last_known_addr { host: "127.11.34.252" port: 43017 } }
20:07:33.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:33.756439 12280 raft_consensus.cc:738] T 00000000000000000000000000000000 P 215376717c434b229999d7279f7e12a6 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 215376717c434b229999d7279f7e12a6, State: Initialized, Role: FOLLOWER
20:07:33.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:33.758123 12280 consensus_queue.cc:260] T 00000000000000000000000000000000 P 215376717c434b229999d7279f7e12a6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e65987b09c81467d8e563fb05fff4cb8" member_type: VOTER last_known_addr { host: "127.11.34.254" port: 38675 } } peers { permanent_uuid: "abed6b5ab4604caa92637633d9f26bbf" member_type: VOTER last_known_addr { host: "127.11.34.253" port: 41205 } } peers { permanent_uuid: "215376717c434b229999d7279f7e12a6" member_type: VOTER last_known_addr { host: "127.11.34.252" port: 43017 } }
20:07:33.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:33.774140 12288 sys_catalog.cc:455] T 00000000000000000000000000000000 P 215376717c434b229999d7279f7e12a6 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e65987b09c81467d8e563fb05fff4cb8" member_type: VOTER last_known_addr { host: "127.11.34.254" port: 38675 } } peers { permanent_uuid: "abed6b5ab4604caa92637633d9f26bbf" member_type: VOTER last_known_addr { host: "127.11.34.253" port: 41205 } } peers { permanent_uuid: "215376717c434b229999d7279f7e12a6" member_type: VOTER last_known_addr { host: "127.11.34.252" port: 43017 } } }
20:07:33.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:33.775007 12288 sys_catalog.cc:458] T 00000000000000000000000000000000 P 215376717c434b229999d7279f7e12a6 [sys.catalog]: This master's current role is: FOLLOWER
20:07:33.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:33.778167 12280 sys_catalog.cc:564] T 00000000000000000000000000000000 P 215376717c434b229999d7279f7e12a6 [sys.catalog]: configured and running, proceeding with master startup.
20:07:33.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:33.810597 12184 heartbeater.cc:344] Connected to a master server at 127.11.34.252:43017
20:07:33.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:33.811167 12184 heartbeater.cc:499] Master 127.11.34.252:43017 was elected leader, sending a full tablet report...
20:07:33.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:33.849792 12301 catalog_manager.cc:1261] Loaded cluster ID: 9cf977073414401480ad287067d54cc3
20:07:33.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:33.850502 12301 catalog_manager.cc:1554] T 00000000000000000000000000000000 P 215376717c434b229999d7279f7e12a6: loading cluster ID for follower catalog manager: success
20:07:33.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:33.866350 12301 catalog_manager.cc:1576] T 00000000000000000000000000000000 P 215376717c434b229999d7279f7e12a6: acquiring CA information for follower catalog manager: success
20:07:33.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:33.851296 12201 master_runner.cc:186] Error getting master registration for 127.11.34.254:38675: Network error: Client connection negotiation failed: client connection to 127.11.34.254:38675: connect: Connection refused (error 111),
20:07:33.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:33.881158 12201 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:07:33.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:33.885026 12301 catalog_manager.cc:1604] T 00000000000000000000000000000000 P 215376717c434b229999d7279f7e12a6: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
20:07:34.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:34.044108 12282 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:07:34.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:34.045096 12282 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:07:34.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:34.046316 12282 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:07:34.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:34.047490 11887 heartbeater.cc:344] Connected to a master server at 127.11.34.252:43017
20:07:34.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:34.053767 12245 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "8544c1a7bcc94996b7d512785fb5655b" instance_seqno: 1757102843794494) as {username='kudu', principal='kudu/127.11.34.193@KRBTEST.COM'} at 127.11.34.193:46907; Asking this server to re-register.
20:07:34.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:34.068645 11887 heartbeater.cc:461] Registering TS with master...
20:07:34.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:34.072538 12245 ts_manager.cc:194] Registered new tserver with Master: 8544c1a7bcc94996b7d512785fb5655b (127.11.34.193:41147)
20:07:34.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:34.118127 12282 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
20:07:34.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:34.118510 12282 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
20:07:34.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:34.119062 12282 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:07:34.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:34.119479 12282 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
20:07:34.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:34.119854 12282 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
20:07:34.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:34.120214 12282 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
20:07:34.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:34.120496 12282 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34155/jwks.json
20:07:34.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:34.199616 12282 master_runner.cc:386] Master server non-default flags:
20:07:34.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:07:34.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.34.212:38019
20:07:34.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:07:34.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:07:34.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/master-0/data
20:07:34.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/master-0/wal
20:07:34.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
20:07:34.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:07:34.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.11.34.254:38675,127.11.34.253:41205,127.11.34.252:43017
20:07:34.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/krb5kdc/kudu.keytab
20:07:34.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.11.34.254
20:07:34.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:07:34.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:07:34.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:07:34.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.34.254:38675
20:07:34.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:07:34.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:07:34.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:07:34.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34155/jwks.json
20:07:34.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:07:34.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:07:34.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:07:34.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:07:34.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:07:34.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/master-0/data/info.pb
20:07:34.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.34.254
20:07:34.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44601
20:07:34.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:07:34.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:07:34.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:07:34.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:07:34.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:07:34.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/master-0/logs
20:07:34.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:07:34.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:07:34.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:07:34.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:07:34.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:07:34.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision e2e116a9cb215058aad161061214d11c617cedac
20:07:34.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:07:34.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 05 Sep 2025 19:43:22 UTC on 5fd53c4cbb9d
20:07:34.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7917
20:07:34.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:07:34.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:34.202346 12282 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:07:34.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:34.204053 12282 file_cache.cc:492] Constructed file cache file cache with capacity 419430
20:07:34.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:34.301505 12311 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:07:34.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:34.679651 12245 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "4d1c197356254990aac35983eb968fe0" instance_seqno: 1757102847380654) as {username='kudu', principal='kudu/127.11.34.194@KRBTEST.COM'} at 127.11.34.194:53161; Asking this server to re-register.
20:07:34.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:34.681866 12037 heartbeater.cc:461] Registering TS with master...
20:07:34.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:34.693645 12245 ts_manager.cc:194] Registered new tserver with Master: 4d1c197356254990aac35983eb968fe0 (127.11.34.194:34845)
20:07:34.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:34.815414 12184 heartbeater.cc:499] Master 127.11.34.252:43017 was elected leader, sending a full tablet report...
20:07:34.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:34.824381 12245 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "d0af648559814b0d9e9af1b19e0eccf1" instance_seqno: 1757102850518857) as {username='kudu', principal='kudu/127.11.34.195@KRBTEST.COM'} at 127.11.34.195:59975; Asking this server to re-register.
20:07:34.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:34.829577 12184 heartbeater.cc:461] Registering TS with master...
20:07:34.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:34.835007 12245 ts_manager.cc:194] Registered new tserver with Master: d0af648559814b0d9e9af1b19e0eccf1 (127.11.34.195:33453)
20:07:35.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:35.466548 12321 raft_consensus.cc:491] T 00000000000000000000000000000000 P 215376717c434b229999d7279f7e12a6 [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:07:35.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:35.467155 12321 raft_consensus.cc:513] T 00000000000000000000000000000000 P 215376717c434b229999d7279f7e12a6 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e65987b09c81467d8e563fb05fff4cb8" member_type: VOTER last_known_addr { host: "127.11.34.254" port: 38675 } } peers { permanent_uuid: "abed6b5ab4604caa92637633d9f26bbf" member_type: VOTER last_known_addr { host: "127.11.34.253" port: 41205 } } peers { permanent_uuid: "215376717c434b229999d7279f7e12a6" member_type: VOTER last_known_addr { host: "127.11.34.252" port: 43017 } }
20:07:35.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:35.482378 12321 leader_election.cc:290] T 00000000000000000000000000000000 P 215376717c434b229999d7279f7e12a6 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers e65987b09c81467d8e563fb05fff4cb8 (127.11.34.254:38675), abed6b5ab4604caa92637633d9f26bbf (127.11.34.253:41205)
20:07:35.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:35.490056 12230 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.11.34.254:38675: connect: Connection refused (error 111)
20:07:35.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:35.505921 12230 leader_election.cc:336] T 00000000000000000000000000000000 P 215376717c434b229999d7279f7e12a6 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer e65987b09c81467d8e563fb05fff4cb8 (127.11.34.254:38675): Network error: Client connection negotiation failed: client connection to 127.11.34.254:38675: connect: Connection refused (error 111)
20:07:35.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:35.514417 12230 leader_election.cc:336] T 00000000000000000000000000000000 P 215376717c434b229999d7279f7e12a6 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer abed6b5ab4604caa92637633d9f26bbf (127.11.34.253:41205): Network error: Client connection negotiation failed: client connection to 127.11.34.253:41205: connect: Connection refused (error 111)
20:07:35.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:35.514879 12230 leader_election.cc:304] T 00000000000000000000000000000000 P 215376717c434b229999d7279f7e12a6 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 215376717c434b229999d7279f7e12a6; no voters: abed6b5ab4604caa92637633d9f26bbf, e65987b09c81467d8e563fb05fff4cb8
20:07:35.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:35.515726 12321 raft_consensus.cc:2747] T 00000000000000000000000000000000 P 215376717c434b229999d7279f7e12a6 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
20:07:36.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:35.678613 12310 debug-util.cc:398] Leaking SignalData structure 0x7b0800037200 after lost signal to thread 12282
20:07:36.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:34.314034 12312 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:07:36.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:36.151003 12282 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 1.840s user 0.544s sys 0.967s
20:07:36.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:36.152320 12282 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 1.841s user 0.544s sys 0.967s
20:07:36.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:36.162597 12329 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:07:36.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:36.162894 12282 server_base.cc:1047] running on GCE node
20:07:36.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:36.164389 12282 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:07:36.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:36.169582 12282 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:07:36.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:36.171497 12282 hybrid_clock.cc:648] HybridClock initialized: now 1757102856171301 us; error 222 us; skew 500 ppm
20:07:36.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:36.172673 12282 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:07:36.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 05 20:07:36 dist-test-slave-x5g2 krb5kdc[11415](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1757102856, etypes {rep=17 tkt=17 ses=17}, kudu/127.11.34.254@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:07:36.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:36.181102 12282 init.cc:377] Logged in from keytab as kudu/127.11.34.254@KRBTEST.COM (short username kudu)
20:07:36.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:36.187057 12282 webserver.cc:480] Webserver started at http://127.11.34.254:44601/ using document root <none> and password file <none>
20:07:36.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:36.188321 12282 fs_manager.cc:362] Metadata directory not provided
20:07:36.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:36.188621 12282 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:07:36.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:36.200166 12282 fs_manager.cc:714] Time spent opening directory manager: real 0.007s user 0.006s sys 0.003s
20:07:36.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:36.206395 12337 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:07:36.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:36.207589 12282 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.001s sys 0.003s
20:07:36.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:36.207960 12282 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/master-0/data,/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/master-0/wal
20:07:36.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e65987b09c81467d8e563fb05fff4cb8"
20:07:36.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-05 20:07:14 on dist-test-slave-x5g2"
20:07:36.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:36.210197 12282 fs_report.cc:389] FS layout report
20:07:36.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:07:36.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/master-0/wal
20:07:36.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/master-0/wal
20:07:36.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/master-0/data/data
20:07:36.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:07:36.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:07:36.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:07:36.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:07:36.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:07:36.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:07:36.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:07:36.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:07:36.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:07:36.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:07:36.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:07:36.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:36.270134 12282 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:07:36.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:36.276672 12282 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:07:36.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:36.277208 12282 kserver.cc:163] Server-wide thread pool size limit: 3276
20:07:36.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:36.371860 12282 rpc_server.cc:307] RPC server started. Bound to: 127.11.34.254:38675
20:07:36.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:36.371937 12388 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.11.34.254:38675 every 8 connection(s)
20:07:36.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:36.375065 12282 server_base.cc:1179] Dumped server information to /tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/master-0/data/info.pb
20:07:36.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:36.375562 11403 external_mini_cluster.cc:1428] Started /tmp/dist-test-taske7evVR/build/tsan/bin/kudu as pid 12282
20:07:36.377 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:07:36.377 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.11.34.253:41205
20:07:36.377 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
20:07:36.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:36.382464 11403 external_mini_cluster.cc:1366] Running /tmp/dist-test-taske7evVR/build/tsan/bin/kudu
20:07:36.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taske7evVR/build/tsan/bin/kudu
20:07:36.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/master-1/wal
20:07:36.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/master-1/data
20:07:36.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:07:36.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:07:36.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:07:36.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:07:36.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:07:36.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:07:36.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/master-1/logs
20:07:36.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/master-1/data/info.pb
20:07:36.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:07:36.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:07:36.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:07:36.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:07:36.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:07:36.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:07:36.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:07:36.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:07:36.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:07:36.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:07:36.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:07:36.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:07:36.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.34.253:41205
20:07:36.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.34.253
20:07:36.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33625
20:07:36.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/krb5kdc/kudu.keytab
20:07:36.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.11.34.253
20:07:36.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:07:36.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:07:36.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:07:36.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
20:07:36.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.34.212:38019
20:07:36.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:07:36.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:07:36.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:07:36.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:07:36.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.11.34.254:38675,127.11.34.253:41205,127.11.34.252:43017
20:07:36.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:07:36.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34155/jwks.json
20:07:36.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
20:07:36.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:07:36.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:36.398065 12389 sys_catalog.cc:263] Verifying existing consensus state
20:07:36.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:36.406970 12389 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P e65987b09c81467d8e563fb05fff4cb8: Bootstrap starting.
20:07:36.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:36.466585 12389 log.cc:826] T 00000000000000000000000000000000 P e65987b09c81467d8e563fb05fff4cb8: Log is configured to *not* fsync() on all Append() calls
20:07:36.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:36.496264 12389 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P e65987b09c81467d8e563fb05fff4cb8: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
20:07:36.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:36.497335 12389 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P e65987b09c81467d8e563fb05fff4cb8: Bootstrap complete.
20:07:36.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:36.531622 12389 raft_consensus.cc:357] T 00000000000000000000000000000000 P e65987b09c81467d8e563fb05fff4cb8 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e65987b09c81467d8e563fb05fff4cb8" member_type: VOTER last_known_addr { host: "127.11.34.254" port: 38675 } } peers { permanent_uuid: "abed6b5ab4604caa92637633d9f26bbf" member_type: VOTER last_known_addr { host: "127.11.34.253" port: 41205 } } peers { permanent_uuid: "215376717c434b229999d7279f7e12a6" member_type: VOTER last_known_addr { host: "127.11.34.252" port: 43017 } }
20:07:36.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:36.535151 12389 raft_consensus.cc:738] T 00000000000000000000000000000000 P e65987b09c81467d8e563fb05fff4cb8 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: e65987b09c81467d8e563fb05fff4cb8, State: Initialized, Role: FOLLOWER
20:07:36.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:36.536243 12389 consensus_queue.cc:260] T 00000000000000000000000000000000 P e65987b09c81467d8e563fb05fff4cb8 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e65987b09c81467d8e563fb05fff4cb8" member_type: VOTER last_known_addr { host: "127.11.34.254" port: 38675 } } peers { permanent_uuid: "abed6b5ab4604caa92637633d9f26bbf" member_type: VOTER last_known_addr { host: "127.11.34.253" port: 41205 } } peers { permanent_uuid: "215376717c434b229999d7279f7e12a6" member_type: VOTER last_known_addr { host: "127.11.34.252" port: 43017 } }
20:07:36.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:36.539826 12393 sys_catalog.cc:455] T 00000000000000000000000000000000 P e65987b09c81467d8e563fb05fff4cb8 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e65987b09c81467d8e563fb05fff4cb8" member_type: VOTER last_known_addr { host: "127.11.34.254" port: 38675 } } peers { permanent_uuid: "abed6b5ab4604caa92637633d9f26bbf" member_type: VOTER last_known_addr { host: "127.11.34.253" port: 41205 } } peers { permanent_uuid: "215376717c434b229999d7279f7e12a6" member_type: VOTER last_known_addr { host: "127.11.34.252" port: 43017 } } }
20:07:36.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:36.541317 12393 sys_catalog.cc:458] T 00000000000000000000000000000000 P e65987b09c81467d8e563fb05fff4cb8 [sys.catalog]: This master's current role is: FOLLOWER
20:07:36.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:36.541672 12389 sys_catalog.cc:564] T 00000000000000000000000000000000 P e65987b09c81467d8e563fb05fff4cb8 [sys.catalog]: configured and running, proceeding with master startup.
20:07:36.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:36.672083 12407 catalog_manager.cc:797] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
20:07:36.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 05 20:07:36 dist-test-slave-x5g2 krb5kdc[11415](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1757102856, etypes {rep=17 tkt=17 ses=17}, kudu/127.11.34.254@KRBTEST.COM for kudu/127.11.34.254@KRBTEST.COM
20:07:36.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:36.729485 12039 heartbeater.cc:344] Connected to a master server at 127.11.34.254:38675
20:07:36.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:36.722214 12282 master_runner.cc:186] Error getting master registration for 127.11.34.253:41205: Network error: Client connection negotiation failed: client connection to 127.11.34.253:41205: connect: Connection refused (error 111),
20:07:36.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:36.746589 12282 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:07:36.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:36.746354 12354 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "4d1c197356254990aac35983eb968fe0" instance_seqno: 1757102847380654) as {username='kudu', principal='kudu/127.11.34.194@KRBTEST.COM'} at 127.11.34.194:44607; Asking this server to re-register.
20:07:36.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:36.748070 12039 heartbeater.cc:461] Registering TS with master...
20:07:36.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:36.750782 12354 ts_manager.cc:194] Registered new tserver with Master: 4d1c197356254990aac35983eb968fe0 (127.11.34.194:34845)
20:07:36.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:36.870451 12391 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:07:36.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:36.871676 12391 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:07:36.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:36.872825 12391 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:07:36.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:36.889407 12188 heartbeater.cc:344] Connected to a master server at 127.11.34.254:38675
20:07:36.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:36.892174 12354 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "d0af648559814b0d9e9af1b19e0eccf1" instance_seqno: 1757102850518857) as {username='kudu', principal='kudu/127.11.34.195@KRBTEST.COM'} at 127.11.34.195:46149; Asking this server to re-register.
20:07:36.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:36.893637 12188 heartbeater.cc:461] Registering TS with master...
20:07:36.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:36.896689 12354 ts_manager.cc:194] Registered new tserver with Master: d0af648559814b0d9e9af1b19e0eccf1 (127.11.34.195:33453)
20:07:36.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:36.914413 12391 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
20:07:36.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:36.914696 12391 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
20:07:36.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:36.915040 12391 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:07:36.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:36.915251 12391 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
20:07:36.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:36.915437 12391 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
20:07:36.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:36.915621 12391 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
20:07:36.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:36.915757 12391 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34155/jwks.json
20:07:36.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:36.984176 12391 master_runner.cc:386] Master server non-default flags:
20:07:36.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:07:36.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.34.212:38019
20:07:36.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:07:36.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:07:36.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/master-1/data
20:07:36.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/master-1/wal
20:07:36.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
20:07:36.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:07:36.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.11.34.254:38675,127.11.34.253:41205,127.11.34.252:43017
20:07:36.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/krb5kdc/kudu.keytab
20:07:36.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.11.34.253
20:07:36.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:07:36.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:07:36.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:07:36.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.34.253:41205
20:07:36.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:07:36.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:07:36.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:07:36.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34155/jwks.json
20:07:36.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:07:36.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:07:36.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:07:36.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:07:36.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:07:36.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/master-1/data/info.pb
20:07:36.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.34.253
20:07:36.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33625
20:07:36.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:07:36.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:07:36.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:07:36.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:07:36.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:07:36.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/master-1/logs
20:07:36.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:07:36.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:07:36.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:07:36.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:07:36.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:07:36.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision e2e116a9cb215058aad161061214d11c617cedac
20:07:36.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:07:36.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 05 Sep 2025 19:43:22 UTC on 5fd53c4cbb9d
20:07:36.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7917
20:07:36.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:07:36.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:36.985921 12391 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:07:36.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:36.987264 12391 file_cache.cc:492] Constructed file cache file cache with capacity 419430
20:07:37.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:37.003151 12420 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:07:37.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:37.113965 12425 raft_consensus.cc:491] T 00000000000000000000000000000000 P 215376717c434b229999d7279f7e12a6 [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:07:37.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:37.115248 12425 raft_consensus.cc:513] T 00000000000000000000000000000000 P 215376717c434b229999d7279f7e12a6 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e65987b09c81467d8e563fb05fff4cb8" member_type: VOTER last_known_addr { host: "127.11.34.254" port: 38675 } } peers { permanent_uuid: "abed6b5ab4604caa92637633d9f26bbf" member_type: VOTER last_known_addr { host: "127.11.34.253" port: 41205 } } peers { permanent_uuid: "215376717c434b229999d7279f7e12a6" member_type: VOTER last_known_addr { host: "127.11.34.252" port: 43017 } }
20:07:37.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:37.127557 12425 leader_election.cc:290] T 00000000000000000000000000000000 P 215376717c434b229999d7279f7e12a6 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers e65987b09c81467d8e563fb05fff4cb8 (127.11.34.254:38675), abed6b5ab4604caa92637633d9f26bbf (127.11.34.253:41205)
20:07:37.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 05 20:07:37 dist-test-slave-x5g2 krb5kdc[11415](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1757102857, etypes {rep=17 tkt=17 ses=17}, kudu/127.11.34.194@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:07:37.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:37.137948 11920 init.cc:295] Successfully reacquired a new kerberos TGT
20:07:37.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 05 20:07:37 dist-test-slave-x5g2 krb5kdc[11415](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1757102853, etypes {rep=17 tkt=17 ses=17}, kudu/127.11.34.193@KRBTEST.COM for kudu/127.11.34.254@KRBTEST.COM
20:07:37.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 05 20:07:37 dist-test-slave-x5g2 krb5kdc[11415](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1757102853, etypes {rep=17 tkt=17 ses=17}, kudu/127.11.34.252@KRBTEST.COM for kudu/127.11.34.254@KRBTEST.COM
20:07:37.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:37.237044 12364 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "215376717c434b229999d7279f7e12a6" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "e65987b09c81467d8e563fb05fff4cb8" is_pre_election: true
20:07:37.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:37.238018 12364 raft_consensus.cc:2466] T 00000000000000000000000000000000 P e65987b09c81467d8e563fb05fff4cb8 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 215376717c434b229999d7279f7e12a6 in term 1.
20:07:37.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:37.239710 11891 heartbeater.cc:344] Connected to a master server at 127.11.34.254:38675
20:07:37.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:37.239784 12230 leader_election.cc:336] T 00000000000000000000000000000000 P 215376717c434b229999d7279f7e12a6 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer abed6b5ab4604caa92637633d9f26bbf (127.11.34.253:41205): Network error: Client connection negotiation failed: client connection to 127.11.34.253:41205: connect: Connection refused (error 111)
20:07:37.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:37.240849 12230 leader_election.cc:304] T 00000000000000000000000000000000 P 215376717c434b229999d7279f7e12a6 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 215376717c434b229999d7279f7e12a6, e65987b09c81467d8e563fb05fff4cb8; no voters: abed6b5ab4604caa92637633d9f26bbf
20:07:37.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:37.241848 12425 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 215376717c434b229999d7279f7e12a6 [term 1 FOLLOWER]: Leader pre-election won for term 2
20:07:37.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:37.242246 12425 raft_consensus.cc:491] T 00000000000000000000000000000000 P 215376717c434b229999d7279f7e12a6 [term 1 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
20:07:37.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:37.242733 12425 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 215376717c434b229999d7279f7e12a6 [term 1 FOLLOWER]: Advancing to term 2
20:07:37.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:37.242862 12354 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "8544c1a7bcc94996b7d512785fb5655b" instance_seqno: 1757102843794494) as {username='kudu', principal='kudu/127.11.34.193@KRBTEST.COM'} at 127.11.34.193:56621; Asking this server to re-register.
20:07:37.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:37.246721 11891 heartbeater.cc:461] Registering TS with master...
20:07:37.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:37.250651 12354 ts_manager.cc:194] Registered new tserver with Master: 8544c1a7bcc94996b7d512785fb5655b (127.11.34.193:41147)
20:07:37.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:37.289603 12425 raft_consensus.cc:513] T 00000000000000000000000000000000 P 215376717c434b229999d7279f7e12a6 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e65987b09c81467d8e563fb05fff4cb8" member_type: VOTER last_known_addr { host: "127.11.34.254" port: 38675 } } peers { permanent_uuid: "abed6b5ab4604caa92637633d9f26bbf" member_type: VOTER last_known_addr { host: "127.11.34.253" port: 41205 } } peers { permanent_uuid: "215376717c434b229999d7279f7e12a6" member_type: VOTER last_known_addr { host: "127.11.34.252" port: 43017 } }
20:07:37.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:37.293167 12425 leader_election.cc:290] T 00000000000000000000000000000000 P 215376717c434b229999d7279f7e12a6 [CANDIDATE]: Term 2 election: Requested vote from peers e65987b09c81467d8e563fb05fff4cb8 (127.11.34.254:38675), abed6b5ab4604caa92637633d9f26bbf (127.11.34.253:41205)
20:07:37.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:37.294502 12364 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "215376717c434b229999d7279f7e12a6" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "e65987b09c81467d8e563fb05fff4cb8"
20:07:37.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:37.295297 12364 raft_consensus.cc:3058] T 00000000000000000000000000000000 P e65987b09c81467d8e563fb05fff4cb8 [term 1 FOLLOWER]: Advancing to term 2
20:07:37.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:37.308066 12230 leader_election.cc:336] T 00000000000000000000000000000000 P 215376717c434b229999d7279f7e12a6 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer abed6b5ab4604caa92637633d9f26bbf (127.11.34.253:41205): Network error: Client connection negotiation failed: client connection to 127.11.34.253:41205: connect: Connection refused (error 111)
20:07:37.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:37.334203 12364 raft_consensus.cc:2466] T 00000000000000000000000000000000 P e65987b09c81467d8e563fb05fff4cb8 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 215376717c434b229999d7279f7e12a6 in term 2.
20:07:37.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:37.337240 12230 leader_election.cc:304] T 00000000000000000000000000000000 P 215376717c434b229999d7279f7e12a6 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 215376717c434b229999d7279f7e12a6, e65987b09c81467d8e563fb05fff4cb8; no voters: abed6b5ab4604caa92637633d9f26bbf
20:07:37.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:37.338256 12425 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 215376717c434b229999d7279f7e12a6 [term 2 FOLLOWER]: Leader election won for term 2
20:07:37.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:37.340606 12425 raft_consensus.cc:695] T 00000000000000000000000000000000 P 215376717c434b229999d7279f7e12a6 [term 2 LEADER]: Becoming Leader. State: Replica: 215376717c434b229999d7279f7e12a6, State: Running, Role: LEADER
20:07:37.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:37.342001 12425 consensus_queue.cc:237] T 00000000000000000000000000000000 P 215376717c434b229999d7279f7e12a6 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e65987b09c81467d8e563fb05fff4cb8" member_type: VOTER last_known_addr { host: "127.11.34.254" port: 38675 } } peers { permanent_uuid: "abed6b5ab4604caa92637633d9f26bbf" member_type: VOTER last_known_addr { host: "127.11.34.253" port: 41205 } } peers { permanent_uuid: "215376717c434b229999d7279f7e12a6" member_type: VOTER last_known_addr { host: "127.11.34.252" port: 43017 } }
20:07:37.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:37.361279 12430 sys_catalog.cc:455] T 00000000000000000000000000000000 P 215376717c434b229999d7279f7e12a6 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 215376717c434b229999d7279f7e12a6. Latest consensus state: current_term: 2 leader_uuid: "215376717c434b229999d7279f7e12a6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e65987b09c81467d8e563fb05fff4cb8" member_type: VOTER last_known_addr { host: "127.11.34.254" port: 38675 } } peers { permanent_uuid: "abed6b5ab4604caa92637633d9f26bbf" member_type: VOTER last_known_addr { host: "127.11.34.253" port: 41205 } } peers { permanent_uuid: "215376717c434b229999d7279f7e12a6" member_type: VOTER last_known_addr { host: "127.11.34.252" port: 43017 } } }
20:07:37.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:37.362186 12430 sys_catalog.cc:458] T 00000000000000000000000000000000 P 215376717c434b229999d7279f7e12a6 [sys.catalog]: This master's current role is: LEADER
20:07:37.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:37.377856 12432 catalog_manager.cc:1477] Loading table and tablet metadata into memory...
20:07:37.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:37.395200 12432 catalog_manager.cc:1486] Initializing Kudu cluster ID...
20:07:37.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:37.405205 12432 catalog_manager.cc:1261] Loaded cluster ID: 9cf977073414401480ad287067d54cc3
20:07:37.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:37.405534 12432 catalog_manager.cc:1497] Initializing Kudu internal certificate authority...
20:07:37.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:37.415658 12432 catalog_manager.cc:1506] Loading token signing keys...
20:07:37.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:37.420300 12432 catalog_manager.cc:5966] T 00000000000000000000000000000000 P 215376717c434b229999d7279f7e12a6: Loaded TSK: 0
20:07:37.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:37.453725 12432 catalog_manager.cc:1516] Initializing in-progress tserver states...
20:07:37.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:37.695919 12407 catalog_manager.cc:1261] Loaded cluster ID: 9cf977073414401480ad287067d54cc3
20:07:37.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:37.696322 12407 catalog_manager.cc:1554] T 00000000000000000000000000000000 P e65987b09c81467d8e563fb05fff4cb8: loading cluster ID for follower catalog manager: success
20:07:37.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:37.716473 12407 catalog_manager.cc:1576] T 00000000000000000000000000000000 P e65987b09c81467d8e563fb05fff4cb8: acquiring CA information for follower catalog manager: success
20:07:37.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:37.734459 12407 catalog_manager.cc:1604] T 00000000000000000000000000000000 P e65987b09c81467d8e563fb05fff4cb8: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
20:07:37.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:37.763504 12230 consensus_peers.cc:489] T 00000000000000000000000000000000 P 215376717c434b229999d7279f7e12a6 -> Peer abed6b5ab4604caa92637633d9f26bbf (127.11.34.253:41205): Couldn't send request to peer abed6b5ab4604caa92637633d9f26bbf. Status: Network error: Client connection negotiation failed: client connection to 127.11.34.253:41205: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
20:07:37.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:37.892851 12364 raft_consensus.cc:1273] T 00000000000000000000000000000000 P e65987b09c81467d8e563fb05fff4cb8 [term 2 FOLLOWER]: Refusing update from remote peer 215376717c434b229999d7279f7e12a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 2 index: 5. (index mismatch)
20:07:37.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:37.895656 12430 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 215376717c434b229999d7279f7e12a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e65987b09c81467d8e563fb05fff4cb8" member_type: VOTER last_known_addr { host: "127.11.34.254" port: 38675 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.001s
20:07:37.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:37.931684 12436 sys_catalog.cc:455] T 00000000000000000000000000000000 P e65987b09c81467d8e563fb05fff4cb8 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 215376717c434b229999d7279f7e12a6. Latest consensus state: current_term: 2 leader_uuid: "215376717c434b229999d7279f7e12a6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e65987b09c81467d8e563fb05fff4cb8" member_type: VOTER last_known_addr { host: "127.11.34.254" port: 38675 } } peers { permanent_uuid: "abed6b5ab4604caa92637633d9f26bbf" member_type: VOTER last_known_addr { host: "127.11.34.253" port: 41205 } } peers { permanent_uuid: "215376717c434b229999d7279f7e12a6" member_type: VOTER last_known_addr { host: "127.11.34.252" port: 43017 } } }
20:07:37.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:37.932687 12436 sys_catalog.cc:458] T 00000000000000000000000000000000 P e65987b09c81467d8e563fb05fff4cb8 [sys.catalog]: This master's current role is: FOLLOWER
20:07:37.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:37.957721 12430 sys_catalog.cc:455] T 00000000000000000000000000000000 P 215376717c434b229999d7279f7e12a6 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "215376717c434b229999d7279f7e12a6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e65987b09c81467d8e563fb05fff4cb8" member_type: VOTER last_known_addr { host: "127.11.34.254" port: 38675 } } peers { permanent_uuid: "abed6b5ab4604caa92637633d9f26bbf" member_type: VOTER last_known_addr { host: "127.11.34.253" port: 41205 } } peers { permanent_uuid: "215376717c434b229999d7279f7e12a6" member_type: VOTER last_known_addr { host: "127.11.34.252" port: 43017 } } }
20:07:37.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:37.961995 12436 sys_catalog.cc:455] T 00000000000000000000000000000000 P e65987b09c81467d8e563fb05fff4cb8 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "215376717c434b229999d7279f7e12a6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e65987b09c81467d8e563fb05fff4cb8" member_type: VOTER last_known_addr { host: "127.11.34.254" port: 38675 } } peers { permanent_uuid: "abed6b5ab4604caa92637633d9f26bbf" member_type: VOTER last_known_addr { host: "127.11.34.253" port: 41205 } } peers { permanent_uuid: "215376717c434b229999d7279f7e12a6" member_type: VOTER last_known_addr { host: "127.11.34.252" port: 43017 } } }
20:07:37.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:37.962729 12436 sys_catalog.cc:458] T 00000000000000000000000000000000 P e65987b09c81467d8e563fb05fff4cb8 [sys.catalog]: This master's current role is: FOLLOWER
20:07:37.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:37.963790 12430 sys_catalog.cc:458] T 00000000000000000000000000000000 P 215376717c434b229999d7279f7e12a6 [sys.catalog]: This master's current role is: LEADER
20:07:38.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:38.710829 12037 heartbeater.cc:499] Master 127.11.34.252:43017 was elected leader, sending a full tablet report...
20:07:38.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:38.883080 12184 heartbeater.cc:499] Master 127.11.34.252:43017 was elected leader, sending a full tablet report...
20:07:38.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:38.900238 12201 master_runner.cc:186] Error getting master registration for 127.11.34.253:41205: Network error: Client connection negotiation failed: client connection to 127.11.34.253:41205: connect: Connection refused (error 111),
20:07:38.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:38.900911 12201 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:07:39.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:39.199960 11887 heartbeater.cc:499] Master 127.11.34.252:43017 was elected leader, sending a full tablet report...
20:07:39.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:38.403985 12418 debug-util.cc:398] Leaking SignalData structure 0x7b0800037200 after lost signal to thread 12391
20:07:39.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:39.665733 12418 kernel_stack_watchdog.cc:198] Thread 12391 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 395ms:
20:07:39.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
20:07:39.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
20:07:39.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
20:07:39.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
20:07:39.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:39.668897 12391 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 2.663s user 0.628s sys 1.236s
20:07:39.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:39.669430 12391 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 2.663s user 0.628s sys 1.237s
20:07:39.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:39.669874 12422 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:07:39.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:39.670598 12421 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2661 milliseconds
20:07:39.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:39.671011 12391 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:07:39.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:37.006390 12419 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:07:39.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:39.682602 12391 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:07:39.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:39.685730 12391 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:07:39.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:39.687180 12391 hybrid_clock.cc:648] HybridClock initialized: now 1757102859686997 us; error 201 us; skew 500 ppm
20:07:39.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:39.688086 12391 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:07:39.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 05 20:07:39 dist-test-slave-x5g2 krb5kdc[11415](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1757102859, etypes {rep=17 tkt=17 ses=17}, kudu/127.11.34.253@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:07:39.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:39.695153 12391 init.cc:377] Logged in from keytab as kudu/127.11.34.253@KRBTEST.COM (short username kudu)
20:07:39.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:39.701248 12391 webserver.cc:480] Webserver started at http://127.11.34.253:33625/ using document root <none> and password file <none>
20:07:39.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:39.702528 12391 fs_manager.cc:362] Metadata directory not provided
20:07:39.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:39.702833 12391 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:07:39.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:39.714104 12391 fs_manager.cc:714] Time spent opening directory manager: real 0.007s user 0.002s sys 0.006s
20:07:39.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:39.720069 12457 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:07:39.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:39.721757 12391 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.004s sys 0.001s
20:07:39.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:39.722173 12391 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/master-1/data,/tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/master-1/wal
20:07:39.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "abed6b5ab4604caa92637633d9f26bbf"
20:07:39.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-05 20:07:17 on dist-test-slave-x5g2"
20:07:39.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:39.725121 12391 fs_report.cc:389] FS layout report
20:07:39.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:07:39.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/master-1/wal
20:07:39.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/master-1/wal
20:07:39.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/master-1/data/data
20:07:39.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:07:39.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:07:39.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:07:39.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:07:39.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:07:39.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:07:39.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:07:39.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:07:39.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:07:39.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:07:39.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:07:39.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:39.806237 12391 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:07:39.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:39.817070 12391 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:07:39.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:39.817624 12391 kserver.cc:163] Server-wide thread pool size limit: 3276
20:07:39.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:39.906278 12391 rpc_server.cc:307] RPC server started. Bound to: 127.11.34.253:41205
20:07:39.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:39.906410 12511 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.11.34.253:41205 every 8 connection(s)
20:07:39.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:39.909227 12391 server_base.cc:1179] Dumped server information to /tmp/dist-test-taske7evVR/test-tmp/mini-kudu-cluster855565418260277073/master-1/data/info.pb
20:07:39.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:39.913260 11403 external_mini_cluster.cc:1428] Started /tmp/dist-test-taske7evVR/build/tsan/bin/kudu as pid 12391
20:07:39.914 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:07:39.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:39.925426 12512 sys_catalog.cc:263] Verifying existing consensus state
20:07:39.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:39.932729 12512 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P abed6b5ab4604caa92637633d9f26bbf: Bootstrap starting.
20:07:39.999 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.11.34.252:43017(127.11.34.252:43017)] connecting to peer
20:07:40.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:40.014544 12516 negotiation.cc:337] Failed RPC negotiation. Trace:
20:07:40.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0905 20:07:39.941102 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.11.34.195:49735 (local address 127.11.34.253:41205)
20:07:40.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0905 20:07:39.963585 (+ 22483us) server_negotiation.cc:207] Beginning negotiation
20:07:40.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0905 20:07:39.963619 (+ 34us) server_negotiation.cc:400] Waiting for connection header
20:07:40.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0905 20:07:39.963694 (+ 75us) server_negotiation.cc:408] Connection header received
20:07:40.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0905 20:07:39.975771 (+ 12077us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
20:07:40.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0905 20:07:39.975866 (+ 95us) server_negotiation.cc:462] Received NEGOTIATE request from client
20:07:40.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0905 20:07:39.976806 (+ 940us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
20:07:40.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0905 20:07:39.980680 (+ 3874us) server_negotiation.cc:227] Negotiated authn=SASL
20:07:40.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0905 20:07:39.983627 (+ 2947us) server_negotiation.cc:366] Received TLS_HANDSHAKE NegotiatePB request
20:07:40.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0905 20:07:39.988057 (+ 4430us) server_negotiation.cc:378] Sending TLS_HANDSHAKE NegotiatePB response
20:07:40.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0905 20:07:39.994754 (+ 6697us) server_negotiation.cc:366] Received TLS_HANDSHAKE NegotiatePB request
20:07:40.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0905 20:07:39.995510 (+ 756us) server_negotiation.cc:658] Negotiated auth-only TLSv1.3 with cipher TLS_AES_128_GCM_SHA256 TLSv1.3 Kx=any Au=any Enc=AESGCM(128) Mac=AEAD
20:07:40.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0905 20:07:39.996190 (+ 680us) server_negotiation.cc:366] Received SASL_INITIATE NegotiatePB request
20:07:40.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0905 20:07:39.996353 (+ 163us) server_negotiation.cc:882] Received SASL_INITIATE request from client
20:07:40.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0905 20:07:39.996415 (+ 62us) server_negotiation.cc:893] Client requested to use mechanism: GSSAPI
20:07:40.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0905 20:07:39.996607 (+ 192us) server_negotiation.cc:923] calling sasl_server_start()
20:07:40.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0905 20:07:40.007113 (+ 10506us) server_negotiation.cc:378] Sending SASL_CHALLENGE NegotiatePB response
20:07:40.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0905 20:07:40.012075 (+ 4962us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.11.34.195:49735: BlockingRecv error: recv got EOF from 127.11.34.195:49735 (error 108)
20:07:40.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":21718,"thread_start_us":22452,"threads_started":1}
20:07:40.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:40.015905 12512 log.cc:826] T 00000000000000000000000000000000 P abed6b5ab4604caa92637633d9f26bbf: Log is configured to *not* fsync() on all Append() calls
20:07:40.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:40.033545 12512 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P abed6b5ab4604caa92637633d9f26bbf: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
20:07:40.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:40.034660 12512 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P abed6b5ab4604caa92637633d9f26bbf: Bootstrap complete.
20:07:40.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:40.063774 12512 raft_consensus.cc:357] T 00000000000000000000000000000000 P abed6b5ab4604caa92637633d9f26bbf [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e65987b09c81467d8e563fb05fff4cb8" member_type: VOTER last_known_addr { host: "127.11.34.254" port: 38675 } } peers { permanent_uuid: "abed6b5ab4604caa92637633d9f26bbf" member_type: VOTER last_known_addr { host: "127.11.34.253" port: 41205 } } peers { permanent_uuid: "215376717c434b229999d7279f7e12a6" member_type: VOTER last_known_addr { host: "127.11.34.252" port: 43017 } }
20:07:40.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:40.067389 12512 raft_consensus.cc:738] T 00000000000000000000000000000000 P abed6b5ab4604caa92637633d9f26bbf [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: abed6b5ab4604caa92637633d9f26bbf, State: Initialized, Role: FOLLOWER
20:07:40.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:40.069096 12512 consensus_queue.cc:260] T 00000000000000000000000000000000 P abed6b5ab4604caa92637633d9f26bbf [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e65987b09c81467d8e563fb05fff4cb8" member_type: VOTER last_known_addr { host: "127.11.34.254" port: 38675 } } peers { permanent_uuid: "abed6b5ab4604caa92637633d9f26bbf" member_type: VOTER last_known_addr { host: "127.11.34.253" port: 41205 } } peers { permanent_uuid: "215376717c434b229999d7279f7e12a6" member_type: VOTER last_known_addr { host: "127.11.34.252" port: 43017 } }
20:07:40.087 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.11.34.254:38675(127.11.34.254:38675)] connecting to peer
20:07:40.089 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.11.34.253:41205(127.11.34.253:41205)] connecting to peer
20:07:40.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:40.089221 12512 sys_catalog.cc:564] T 00000000000000000000000000000000 P abed6b5ab4604caa92637633d9f26bbf [sys.catalog]: configured and running, proceeding with master startup.
20:07:40.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:40.086858 12518 sys_catalog.cc:455] T 00000000000000000000000000000000 P abed6b5ab4604caa92637633d9f26bbf [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e65987b09c81467d8e563fb05fff4cb8" member_type: VOTER last_known_addr { host: "127.11.34.254" port: 38675 } } peers { permanent_uuid: "abed6b5ab4604caa92637633d9f26bbf" member_type: VOTER last_known_addr { host: "127.11.34.253" port: 41205 } } peers { permanent_uuid: "215376717c434b229999d7279f7e12a6" member_type: VOTER last_known_addr { host: "127.11.34.252" port: 43017 } } }
20:07:40.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:40.092558 12518 sys_catalog.cc:458] T 00000000000000000000000000000000 P abed6b5ab4604caa92637633d9f26bbf [sys.catalog]: This master's current role is: FOLLOWER
20:07:40.167 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.11.34.254:38675(127.11.34.254:38675)] Successfully connected to peer
20:07:40.168 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.11.34.252:43017(127.11.34.252:43017)] Successfully connected to peer
20:07:40.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:40.171921 12531 catalog_manager.cc:1261] Loaded cluster ID: 9cf977073414401480ad287067d54cc3
20:07:40.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:40.172212 12531 catalog_manager.cc:1554] T 00000000000000000000000000000000 P abed6b5ab4604caa92637633d9f26bbf: loading cluster ID for follower catalog manager: success
20:07:40.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 05 20:07:40 dist-test-slave-x5g2 krb5kdc[11415](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1757102860, etypes {rep=17 tkt=17 ses=17}, kudu/127.11.34.195@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:07:40.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:40.177618 12067 init.cc:295] Successfully reacquired a new kerberos TGT
20:07:40.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:40.188925 12531 catalog_manager.cc:1576] T 00000000000000000000000000000000 P abed6b5ab4604caa92637633d9f26bbf: acquiring CA information for follower catalog manager: success
20:07:40.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 05 20:07:40 dist-test-slave-x5g2 krb5kdc[11415](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1757102859, etypes {rep=17 tkt=17 ses=17}, kudu/127.11.34.253@KRBTEST.COM for kudu/127.11.34.254@KRBTEST.COM
20:07:40.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:40.247920 12531 catalog_manager.cc:1604] T 00000000000000000000000000000000 P abed6b5ab4604caa92637633d9f26bbf: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
20:07:40.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:40.266263 12487 raft_consensus.cc:3058] T 00000000000000000000000000000000 P abed6b5ab4604caa92637633d9f26bbf [term 1 FOLLOWER]: Advancing to term 2
20:07:40.313 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.11.34.253:41205(127.11.34.253:41205)] Successfully connected to peer
20:07:40.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:40.331061 12543 sys_catalog.cc:455] T 00000000000000000000000000000000 P abed6b5ab4604caa92637633d9f26bbf [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "215376717c434b229999d7279f7e12a6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e65987b09c81467d8e563fb05fff4cb8" member_type: VOTER last_known_addr { host: "127.11.34.254" port: 38675 } } peers { permanent_uuid: "abed6b5ab4604caa92637633d9f26bbf" member_type: VOTER last_known_addr { host: "127.11.34.253" port: 41205 } } peers { permanent_uuid: "215376717c434b229999d7279f7e12a6" member_type: VOTER last_known_addr { host: "127.11.34.252" port: 43017 } } }
20:07:40.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:40.331877 12543 sys_catalog.cc:458] T 00000000000000000000000000000000 P abed6b5ab4604caa92637633d9f26bbf [sys.catalog]: This master's current role is: FOLLOWER
20:07:40.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:40.334605 12518 sys_catalog.cc:455] T 00000000000000000000000000000000 P abed6b5ab4604caa92637633d9f26bbf [sys.catalog]: SysCatalogTable state changed. Reason: New leader 215376717c434b229999d7279f7e12a6. Latest consensus state: current_term: 2 leader_uuid: "215376717c434b229999d7279f7e12a6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e65987b09c81467d8e563fb05fff4cb8" member_type: VOTER last_known_addr { host: "127.11.34.254" port: 38675 } } peers { permanent_uuid: "abed6b5ab4604caa92637633d9f26bbf" member_type: VOTER last_known_addr { host: "127.11.34.253" port: 41205 } } peers { permanent_uuid: "215376717c434b229999d7279f7e12a6" member_type: VOTER last_known_addr { host: "127.11.34.252" port: 43017 } } }
20:07:40.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:40.335314 12518 sys_catalog.cc:458] T 00000000000000000000000000000000 P abed6b5ab4604caa92637633d9f26bbf [sys.catalog]: This master's current role is: FOLLOWER
20:07:40.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:40.336038 11888 heartbeater.cc:344] Connected to a master server at 127.11.34.253:41205
20:07:40.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:40.342535 12476 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "8544c1a7bcc94996b7d512785fb5655b" instance_seqno: 1757102843794494) as {username='kudu', principal='kudu/127.11.34.193@KRBTEST.COM'} at 127.11.34.193:36067; Asking this server to re-register.
20:07:40.343 [WARN - kudu-nio-0] (SecurityContext.java:206) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
20:07:40.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:40.344496 11888 heartbeater.cc:461] Registering TS with master...
20:07:40.347 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.11.34.254:38675(127.11.34.254:38675)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
20:07:40.348 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.11.34.252:43017(127.11.34.252:43017)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
20:07:40.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:40.348382 12476 ts_manager.cc:194] Registered new tserver with Master: 8544c1a7bcc94996b7d512785fb5655b (127.11.34.193:41147)
20:07:40.350 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.11.34.254:38675: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
20:07:40.351 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.11.34.252:43017: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
20:07:40.355 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.11.34.254:38675(127.11.34.254:38675)] handling channelInactive
20:07:40.356 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.11.34.253:41205(127.11.34.253:41205)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
20:07:40.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:40.356053 12535 negotiation.cc:337] Failed RPC negotiation. Trace:
20:07:40.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0905 20:07:40.160334 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:58866 (local address 127.11.34.252:43017)
20:07:40.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0905 20:07:40.166859 (+ 6525us) server_negotiation.cc:207] Beginning negotiation
20:07:40.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0905 20:07:40.166886 (+ 27us) server_negotiation.cc:400] Waiting for connection header
20:07:40.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0905 20:07:40.238136 (+ 71250us) server_negotiation.cc:408] Connection header received
20:07:40.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0905 20:07:40.310659 (+ 72523us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
20:07:40.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0905 20:07:40.310690 (+ 31us) server_negotiation.cc:462] Received NEGOTIATE request from client
20:07:40.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0905 20:07:40.310925 (+ 235us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
20:07:40.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0905 20:07:40.311094 (+ 169us) server_negotiation.cc:227] Negotiated authn=SASL
20:07:40.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0905 20:07:40.354559 (+ 43465us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:58866: BlockingRecv error: recv got EOF from 127.0.0.1:58866 (error 108)
20:07:40.357 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.11.34.253:41205: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
20:07:40.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":5906,"thread_start_us":5652,"threads_started":1}
20:07:40.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:40.358649 12536 negotiation.cc:337] Failed RPC negotiation. Trace:
20:07:40.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0905 20:07:40.164726 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:55202 (local address 127.11.34.254:38675)
20:07:40.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0905 20:07:40.172882 (+ 8156us) server_negotiation.cc:207] Beginning negotiation
20:07:40.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0905 20:07:40.172910 (+ 28us) server_negotiation.cc:400] Waiting for connection header
20:07:40.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0905 20:07:40.235637 (+ 62727us) server_negotiation.cc:408] Connection header received
20:07:40.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0905 20:07:40.308899 (+ 73262us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
20:07:40.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0905 20:07:40.308930 (+ 31us) server_negotiation.cc:462] Received NEGOTIATE request from client
20:07:40.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0905 20:07:40.309168 (+ 238us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
20:07:40.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0905 20:07:40.309405 (+ 237us) server_negotiation.cc:227] Negotiated authn=SASL
20:07:40.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0905 20:07:40.357044 (+ 47639us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:55202: BlockingRecv error: recv got EOF from 127.0.0.1:55202 (error 108)
20:07:40.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":7508,"thread_start_us":7227,"threads_started":1}
20:07:40.362 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1523) Notify RPC KuduRpc(method=ListTabletServers, tablet=null, attempt=1, TimeoutTracker(timeout=30000, elapsed=442), Traces: [0ms] refreshing cache from master, [45ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.34.252:43017, [162ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.34.254:38675, [164ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.34.253:41205, [424ms] Sub RPC ConnectToMaster: received response from server master-127.11.34.254:38675: Not authorized: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, [425ms] Sub RPC ConnectToMaster: received response from server master-127.11.34.252:43017: Not authorized: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, [432ms] Sub RPC ConnectToMaster: received response from server master-127.11.34.253:41205: Not authorized: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, deferred=null) after lookup exception
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.11.34.252:43017,127.11.34.254:38675,127.11.34.253:41205). Exceptions received: [org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268) [main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) [main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387) [main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376) [main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) [async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) [async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315) [async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286) [async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) [async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) [async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) [main/:?]
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331) [main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256) [main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) [main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) [main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) [main/:?]
at org.apache.kudu.client.Connection.cleanup(Connection.java:737) [main/:?]
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455) [main/:?]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) [netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) [netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) [netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) [netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) [netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_275]
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.11.34.252:43017,127.11.34.254:38675,127.11.34.253:41205). Exceptions received: [org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
20:07:40.371 [DEBUG - kudu-nio-0] (KuduRpc.java:260) Handling a callback on RPC KuduRpc(method=ListTabletServers, tablet=null, attempt=1, TimeoutTracker(timeout=30000, elapsed=452), Traces: [0ms] refreshing cache from master, [45ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.34.252:43017, [162ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.34.254:38675, [164ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.34.253:41205, [424ms] Sub RPC ConnectToMaster: received response from server master-127.11.34.254:38675: Not authorized: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, [425ms] Sub RPC ConnectToMaster: received response from server master-127.11.34.252:43017: Not authorized: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, [432ms] Sub RPC ConnectToMaster: received response from server master-127.11.34.253:41205: Not authorized: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, deferred=null) with no deferred attached!
20:07:40.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:40.373338 12516 negotiation.cc:337] Failed RPC negotiation. Trace:
20:07:40.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0905 20:07:40.163575 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:40602 (local address 127.11.34.253:41205)
20:07:40.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0905 20:07:40.164328 (+ 753us) server_negotiation.cc:207] Beginning negotiation
20:07:40.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0905 20:07:40.164354 (+ 26us) server_negotiation.cc:400] Waiting for connection header
20:07:40.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0905 20:07:40.314047 (+149693us) server_negotiation.cc:408] Connection header received
20:07:40.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0905 20:07:40.315041 (+ 994us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
20:07:40.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0905 20:07:40.315067 (+ 26us) server_negotiation.cc:462] Received NEGOTIATE request from client
20:07:40.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0905 20:07:40.315290 (+ 223us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
20:07:40.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0905 20:07:40.315435 (+ 145us) server_negotiation.cc:227] Negotiated authn=SASL
20:07:40.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0905 20:07:40.372372 (+ 56937us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:40602: BlockingRecv error: recv got EOF from 127.0.0.1:40602 (error 108)
20:07:40.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":160}
20:07:40.375 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.11.34.253:41205(127.11.34.253:41205)] disconnecting while in state TERMINATED
20:07:40.376 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.11.34.254:38675(127.11.34.254:38675)] disconnecting while in state TERMINATED
20:07:40.376 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.11.34.252:43017(127.11.34.252:43017)] disconnecting while in state TERMINATED
20:07:40.383 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.11.34.252:43017(127.11.34.252:43017)] handling channelInactive
20:07:40.384 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.11.34.253:41205(127.11.34.253:41205)] handling channelInactive
20:07:40.391 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2704) Releasing all remaining resources
20:07:40.398 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
20:07:40.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:40.405465 11403 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taske7evVR/build/tsan/bin/kudu with pid 11745
20:07:40.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:40.456676 11403 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taske7evVR/build/tsan/bin/kudu with pid 11900
20:07:40.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:40.496909 11403 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taske7evVR/build/tsan/bin/kudu with pid 12051
20:07:40.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:40.532968 11403 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taske7evVR/build/tsan/bin/kudu with pid 12282
20:07:40.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:40.573724 11403 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taske7evVR/build/tsan/bin/kudu with pid 12391
20:07:40.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:40.603593 11403 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taske7evVR/build/tsan/bin/kudu with pid 12201
20:07:41.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-09-05T20:07:41Z chronyd exiting
20:07:42.851 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
20:07:42.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:42.852922 10670 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taske7evVR/build/tsan/bin/kudu with pid 10954
20:07:42.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:42.879083 10670 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taske7evVR/build/tsan/bin/kudu with pid 11116
20:07:42.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:42.904536 10670 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taske7evVR/build/tsan/bin/kudu with pid 11261
20:07:42.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:42.932610 10670 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taske7evVR/build/tsan/bin/kudu with pid 10682
20:07:42.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250905 20:07:42.961381 10776 connection.cc:537] client connection to 127.10.107.190:42101 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
20:07:42.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:42.961678 10670 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taske7evVR/build/tsan/bin/kudu with pid 10752
20:07:42.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250905 20:07:42.991603 10670 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taske7evVR/build/tsan/bin/kudu with pid 10827
20:07:44.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-09-05T20:07:44Z chronyd exiting
20:07:45.169 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.11.34.252:43017,127.11.34.254:38675,127.11.34.253:41205). Exceptions received: [org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available]
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.cleanup(Connection.java:737) ~[main/:?]
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455) ~[main/:?]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.11.34.252:43017,127.11.34.254:38675,127.11.34.253:41205). Exceptions received: [org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available]
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:748)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
20:07:45.585 [ERROR - main] (RetryRule.java:253) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: giving up after 1 attempts
20:07:45.586 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.11.34.252:43017,127.11.34.254:38675,127.11.34.253:41205). Exceptions received: [org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available]
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.cleanup(Connection.java:737) ~[main/:?]
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455) ~[main/:?]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.11.34.252:43017,127.11.34.254:38675,127.11.34.253:41205). Exceptions received: [org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available]
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:748)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more