08:07:08.799 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
08:07:08.810 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskcrgkbR/test-tmp
08:07:08.813 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
08:07:08.817 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskcrgkbR/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
08:07:09.063 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster3944160619542668700"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
08:07:09.223 [DEBUG - main] (MiniKuduCluster.java:195) Response:
08:07:09.224 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
08:07:09.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:09.228139 7815 mini_oidc.cc:150] Starting JWKS server
08:07:09.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:09.231398 7815 webserver.cc:492] Webserver started at http://127.0.0.1:45001/ using document root <none> and password file <none>
08:07:09.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:09.231818 7815 mini_oidc.cc:186] Starting OIDC Discovery server
08:07:09.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:09.233544 7815 webserver.cc:492] Webserver started at http://127.0.0.1:33611/ using document root <none> and password file <none>
08:07:09.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-10-23T08:07:09Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
08:07:09.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-10-23T08:07:09Z Disabled control of system clock
08:07:09.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:09.293558 7815 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskcrgkbR/build/tsan/bin/kudu
08:07:09.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskcrgkbR/build/tsan/bin/kudu
08:07:09.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster3944160619542668700/master-0/wal
08:07:09.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster3944160619542668700/master-0/data
08:07:09.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:07:09.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:07:09.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:07:09.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:07:09.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:07:09.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:07:09.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster3944160619542668700/master-0/logs
08:07:09.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster3944160619542668700/master-0/data/info.pb
08:07:09.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:07:09.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:07:09.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:07:09.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:07:09.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:07:09.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:07:09.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:07:09.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:07:09.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:07:09.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:07:09.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:07:09.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:07:09.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.161.254:39651
08:07:09.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.161.254
08:07:09.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:07:09.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.161.212:38263
08:07:09.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:07:09.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:07:09.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:07:09.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:07:09.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.161.254:39651,127.7.161.253:44339,127.7.161.252:36527
08:07:09.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:07:09.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45001/jwks.json with env {}
08:07:09.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:09.588574 7827 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:07:09.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:09.589133 7827 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:07:09.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:09.589702 7827 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:07:09.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:09.621551 7827 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
08:07:09.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:09.621838 7827 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:07:09.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:09.622028 7827 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
08:07:09.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:09.622272 7827 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
08:07:09.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:09.622534 7827 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
08:07:09.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:09.622679 7827 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:45001/jwks.json
08:07:09.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:09.659339 7827 master_runner.cc:386] Master server non-default flags:
08:07:09.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:07:09.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.161.212:38263
08:07:09.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:07:09.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:07:09.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster3944160619542668700/master-0/data
08:07:09.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster3944160619542668700/master-0/wal
08:07:09.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:07:09.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.161.254:39651,127.7.161.253:44339,127.7.161.252:36527
08:07:09.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:07:09.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:07:09.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:07:09.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.161.254:39651
08:07:09.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:07:09.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:07:09.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:07:09.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45001/jwks.json
08:07:09.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:07:09.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:07:09.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster3944160619542668700/master-0/data/info.pb
08:07:09.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.161.254
08:07:09.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:07:09.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:07:09.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:07:09.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:07:09.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:07:09.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:07:09.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster3944160619542668700/master-0/logs
08:07:09.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:07:09.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:07:09.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:07:09.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:07:09.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:07:09.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 18b1930e830609248ee2e26d6216d073ff40ac5d
08:07:09.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:07:09.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 23 Oct 2025 07:43:21 UTC on 5fd53c4cbb9d
08:07:09.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8677
08:07:09.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:07:09.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:09.660737 7827 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:07:09.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:09.662424 7827 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:07:09.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:09.674075 7833 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:07:11.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:11.077589 7832 debug-util.cc:398] Leaking SignalData structure 0x7b0800037520 after lost signal to thread 7827
08:07:11.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:11.460058 7832 kernel_stack_watchdog.cc:198] Thread 7827 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 400ms:
08:07:11.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
08:07:11.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
08:07:11.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
08:07:11.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
08:07:11.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:09.674808 7834 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:07:11.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:11.461624 7827 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.787s user 0.000s sys 0.001s
08:07:11.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:11.461995 7827 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.788s user 0.000s sys 0.002s
08:07:11.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:11.462608 7835 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1785 milliseconds
08:07:11.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:11.464025 7836 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:07:11.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:11.464860 7827 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
08:07:11.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:11.467751 7827 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:07:11.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:11.470216 7827 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:07:11.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:11.471560 7827 hybrid_clock.cc:648] HybridClock initialized: now 1761206831471495 us; error 76 us; skew 500 ppm
08:07:11.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:11.472424 7827 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:07:11.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:11.478817 7827 webserver.cc:492] Webserver started at http://127.7.161.254:37713/ using document root <none> and password file <none>
08:07:11.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:11.479776 7827 fs_manager.cc:362] Metadata directory not provided
08:07:11.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:11.479987 7827 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:07:11.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:11.480588 7827 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
08:07:11.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:11.484753 7827 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster3944160619542668700/master-0/data/instance:
08:07:11.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e08a5f066ba34c6f98c017e1304905bb"
08:07:11.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-23 08:07:11 on dist-test-slave-wxqn"
08:07:11.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:11.486085 7827 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster3944160619542668700/master-0/wal/instance:
08:07:11.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e08a5f066ba34c6f98c017e1304905bb"
08:07:11.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-23 08:07:11 on dist-test-slave-wxqn"
08:07:11.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:11.493336 7827 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.007s sys 0.001s
08:07:11.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:11.499084 7843 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:07:11.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:11.500221 7827 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.000s
08:07:11.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:11.500600 7827 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster3944160619542668700/master-0/data,/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster3944160619542668700/master-0/wal
08:07:11.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e08a5f066ba34c6f98c017e1304905bb"
08:07:11.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-23 08:07:11 on dist-test-slave-wxqn"
08:07:11.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:11.500979 7827 fs_report.cc:389] FS layout report
08:07:11.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:07:11.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster3944160619542668700/master-0/wal
08:07:11.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster3944160619542668700/master-0/wal
08:07:11.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster3944160619542668700/master-0/data/data
08:07:11.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:07:11.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:07:11.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:07:11.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:07:11.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:07:11.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:07:11.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:07:11.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:07:11.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:07:11.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:07:11.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:07:11.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:11.552184 7827 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:07:11.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:11.553810 7827 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:07:11.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:11.554319 7827 kserver.cc:163] Server-wide thread pool size limit: 3276
08:07:11.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:11.626109 7827 rpc_server.cc:307] RPC server started. Bound to: 127.7.161.254:39651
08:07:11.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:11.626188 7894 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.7.161.254:39651 every 8 connection(s)
08:07:11.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:11.628954 7827 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster3944160619542668700/master-0/data/info.pb
08:07:11.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:11.635658 7895 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
08:07:11.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:11.639518 7815 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskcrgkbR/build/tsan/bin/kudu as pid 7827
08:07:11.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:11.639971 7815 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster3944160619542668700/master-0/wal/instance
08:07:11.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:11.643338 7815 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskcrgkbR/build/tsan/bin/kudu
08:07:11.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskcrgkbR/build/tsan/bin/kudu
08:07:11.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster3944160619542668700/master-1/wal
08:07:11.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster3944160619542668700/master-1/data
08:07:11.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:07:11.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:07:11.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:07:11.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:07:11.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:07:11.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:07:11.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster3944160619542668700/master-1/logs
08:07:11.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster3944160619542668700/master-1/data/info.pb
08:07:11.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:07:11.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:07:11.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:07:11.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:07:11.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:07:11.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:07:11.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:07:11.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:07:11.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:07:11.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:07:11.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:07:11.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:07:11.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.161.253:44339
08:07:11.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.161.253
08:07:11.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:07:11.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.161.212:38263
08:07:11.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:07:11.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:07:11.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:07:11.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:07:11.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.161.254:39651,127.7.161.253:44339,127.7.161.252:36527
08:07:11.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:07:11.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45001/jwks.json with env {}
08:07:11.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:11.646893 7895 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.161.254" port: 39651 } has no permanent_uuid. Determining permanent_uuid...
08:07:11.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:11.679569 7895 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.161.253" port: 44339 } has no permanent_uuid. Determining permanent_uuid...
08:07:11.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:11.681942 7847 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.161.253:44339: connect: Connection refused (error 111)
08:07:11.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:11.686477 7895 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.7.161.253:44339: Network error: Client connection negotiation failed: client connection to 127.7.161.253:44339: connect: Connection refused (error 111)
08:07:11.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:11.735924 7895 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.161.253" port: 44339 } attempt: 1
08:07:11.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:11.740422 7895 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.7.161.253:44339: Network error: Client connection negotiation failed: client connection to 127.7.161.253:44339: connect: Connection refused (error 111)
08:07:11.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:11.808851 7895 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.161.253" port: 44339 } attempt: 2
08:07:11.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:11.813809 7895 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.7.161.253:44339: Network error: Client connection negotiation failed: client connection to 127.7.161.253:44339: connect: Connection refused (error 111)
08:07:11.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:11.905335 7895 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.161.253" port: 44339 } attempt: 3
08:07:11.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:11.909363 7895 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.7.161.253:44339: Network error: Client connection negotiation failed: client connection to 127.7.161.253:44339: connect: Connection refused (error 111)
08:07:11.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:11.957914 7897 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:07:11.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:11.958559 7897 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:07:11.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:11.959204 7897 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:07:11.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:11.989939 7897 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
08:07:11.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:11.990300 7897 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:07:11.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:11.990644 7897 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
08:07:11.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:11.991004 7897 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
08:07:11.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:11.991391 7897 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
08:07:11.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:11.991641 7897 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:45001/jwks.json
08:07:12.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:12.027761 7897 master_runner.cc:386] Master server non-default flags:
08:07:12.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:07:12.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.161.212:38263
08:07:12.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:07:12.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:07:12.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster3944160619542668700/master-1/data
08:07:12.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster3944160619542668700/master-1/wal
08:07:12.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:07:12.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.161.254:39651,127.7.161.253:44339,127.7.161.252:36527
08:07:12.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:07:12.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:07:12.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:07:12.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.161.253:44339
08:07:12.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:07:12.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:07:12.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:07:12.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45001/jwks.json
08:07:12.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:07:12.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:07:12.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster3944160619542668700/master-1/data/info.pb
08:07:12.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.161.253
08:07:12.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:07:12.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:07:12.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:07:12.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:07:12.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:07:12.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:07:12.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster3944160619542668700/master-1/logs
08:07:12.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:07:12.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:07:12.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:07:12.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:07:12.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:07:12.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 18b1930e830609248ee2e26d6216d073ff40ac5d
08:07:12.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:07:12.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 23 Oct 2025 07:43:21 UTC on 5fd53c4cbb9d
08:07:12.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8677
08:07:12.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:07:12.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:12.029598 7897 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:07:12.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:12.031961 7897 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:07:12.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:12.043203 7906 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:07:12.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:12.052738 7895 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.161.253" port: 44339 } attempt: 4
08:07:12.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:12.056661 7895 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.7.161.253:44339: Network error: Client connection negotiation failed: client connection to 127.7.161.253:44339: connect: Connection refused (error 111)
08:07:12.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:12.356108 7895 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.161.253" port: 44339 } attempt: 5
08:07:12.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:12.363258 7895 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.7.161.253:44339: Network error: Client connection negotiation failed: client connection to 127.7.161.253:44339: connect: Connection refused (error 111)
08:07:12.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:12.910878 7895 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.161.253" port: 44339 } attempt: 6
08:07:12.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:12.924160 7895 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.7.161.253:44339: Network error: Client connection negotiation failed: client connection to 127.7.161.253:44339: connect: Connection refused (error 111)
08:07:13.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:12.045885 7907 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:07:13.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:12.048470 7909 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:07:13.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:13.243985 7908 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1195 milliseconds
08:07:13.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:13.244102 7897 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
08:07:13.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:13.245292 7897 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:07:13.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:13.248272 7897 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:07:13.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:13.249733 7897 hybrid_clock.cc:648] HybridClock initialized: now 1761206833249707 us; error 73 us; skew 500 ppm
08:07:13.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:13.250651 7897 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:07:13.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:13.256644 7897 webserver.cc:492] Webserver started at http://127.7.161.253:33521/ using document root <none> and password file <none>
08:07:13.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:13.257474 7897 fs_manager.cc:362] Metadata directory not provided
08:07:13.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:13.257666 7897 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:07:13.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:13.258025 7897 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
08:07:13.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:13.262413 7897 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster3944160619542668700/master-1/data/instance:
08:07:13.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c0a6c76326f44dd4a06424b1bb421f76"
08:07:13.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-23 08:07:13 on dist-test-slave-wxqn"
08:07:13.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:13.263460 7897 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster3944160619542668700/master-1/wal/instance:
08:07:13.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c0a6c76326f44dd4a06424b1bb421f76"
08:07:13.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-23 08:07:13 on dist-test-slave-wxqn"
08:07:13.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:13.272092 7897 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.005s sys 0.006s
08:07:13.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:13.277648 7918 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:07:13.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:13.278712 7897 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.002s
08:07:13.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:13.278990 7897 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster3944160619542668700/master-1/data,/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster3944160619542668700/master-1/wal
08:07:13.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c0a6c76326f44dd4a06424b1bb421f76"
08:07:13.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-23 08:07:13 on dist-test-slave-wxqn"
08:07:13.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:13.279277 7897 fs_report.cc:389] FS layout report
08:07:13.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:07:13.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster3944160619542668700/master-1/wal
08:07:13.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster3944160619542668700/master-1/wal
08:07:13.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster3944160619542668700/master-1/data/data
08:07:13.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:07:13.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:07:13.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:07:13.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:07:13.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:07:13.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:07:13.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:07:13.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:07:13.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:07:13.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:07:13.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:07:13.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:13.327049 7897 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:07:13.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:13.328445 7897 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:07:13.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:13.328871 7897 kserver.cc:163] Server-wide thread pool size limit: 3276
08:07:13.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:13.400082 7897 rpc_server.cc:307] RPC server started. Bound to: 127.7.161.253:44339
08:07:13.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:13.400211 7969 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.7.161.253:44339 every 8 connection(s)
08:07:13.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:13.403044 7897 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster3944160619542668700/master-1/data/info.pb
08:07:13.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:13.404877 7815 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskcrgkbR/build/tsan/bin/kudu as pid 7897
08:07:13.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:13.405416 7815 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster3944160619542668700/master-1/wal/instance
08:07:13.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:13.407965 7815 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskcrgkbR/build/tsan/bin/kudu
08:07:13.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskcrgkbR/build/tsan/bin/kudu
08:07:13.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster3944160619542668700/master-2/wal
08:07:13.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster3944160619542668700/master-2/data
08:07:13.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:07:13.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:07:13.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:07:13.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:07:13.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:07:13.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:07:13.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster3944160619542668700/master-2/logs
08:07:13.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster3944160619542668700/master-2/data/info.pb
08:07:13.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:07:13.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:07:13.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:07:13.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:07:13.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:07:13.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:07:13.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:07:13.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:07:13.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:07:13.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:07:13.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:07:13.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:07:13.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.161.252:36527
08:07:13.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.161.252
08:07:13.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:07:13.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.161.212:38263
08:07:13.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:07:13.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:07:13.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:07:13.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:07:13.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.161.254:39651,127.7.161.253:44339,127.7.161.252:36527
08:07:13.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:07:13.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45001/jwks.json with env {}
08:07:13.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:13.409718 7970 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
08:07:13.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:13.422596 7970 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.161.254" port: 39651 } has no permanent_uuid. Determining permanent_uuid...
08:07:13.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:13.443646 7970 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.161.253" port: 44339 } has no permanent_uuid. Determining permanent_uuid...
08:07:13.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:13.463074 7970 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.161.252" port: 36527 } has no permanent_uuid. Determining permanent_uuid...
08:07:13.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:13.465651 7920 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.161.252:36527: connect: Connection refused (error 111)
08:07:13.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:13.469472 7970 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.7.161.252:36527: Network error: Client connection negotiation failed: client connection to 127.7.161.252:36527: connect: Connection refused (error 111)
08:07:13.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:13.518962 7970 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.161.252" port: 36527 } attempt: 1
08:07:13.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:13.522810 7970 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.7.161.252:36527: Network error: Client connection negotiation failed: client connection to 127.7.161.252:36527: connect: Connection refused (error 111)
08:07:13.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:13.591359 7970 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.161.252" port: 36527 } attempt: 2
08:07:13.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:13.595530 7970 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.7.161.252:36527: Network error: Client connection negotiation failed: client connection to 127.7.161.252:36527: connect: Connection refused (error 111)
08:07:13.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:13.686982 7970 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.161.252" port: 36527 } attempt: 3
08:07:13.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:13.691335 7970 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.7.161.252:36527: Network error: Client connection negotiation failed: client connection to 127.7.161.252:36527: connect: Connection refused (error 111)
08:07:13.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:13.723656 7972 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:07:13.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:13.724179 7972 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:07:13.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:13.724733 7972 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:07:13.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:13.756965 7972 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
08:07:13.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:13.757244 7972 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:07:13.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:13.757436 7972 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
08:07:13.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:13.757635 7972 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
08:07:13.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:13.757812 7972 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
08:07:13.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:13.757958 7972 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:45001/jwks.json
08:07:13.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:13.793802 7972 master_runner.cc:386] Master server non-default flags:
08:07:13.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:07:13.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.161.212:38263
08:07:13.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:07:13.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:07:13.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster3944160619542668700/master-2/data
08:07:13.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster3944160619542668700/master-2/wal
08:07:13.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:07:13.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.161.254:39651,127.7.161.253:44339,127.7.161.252:36527
08:07:13.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:07:13.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:07:13.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:07:13.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.161.252:36527
08:07:13.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:07:13.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:07:13.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:07:13.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45001/jwks.json
08:07:13.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:07:13.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:07:13.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster3944160619542668700/master-2/data/info.pb
08:07:13.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.161.252
08:07:13.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:07:13.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:07:13.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:07:13.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:07:13.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:07:13.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:07:13.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster3944160619542668700/master-2/logs
08:07:13.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:07:13.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:07:13.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:07:13.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:07:13.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:07:13.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 18b1930e830609248ee2e26d6216d073ff40ac5d
08:07:13.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:07:13.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 23 Oct 2025 07:43:21 UTC on 5fd53c4cbb9d
08:07:13.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8677
08:07:13.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:07:13.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:13.795269 7972 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:07:13.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:13.797379 7972 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:07:13.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:13.810307 7982 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:07:13.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:13.834800 7970 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.161.252" port: 36527 } attempt: 4
08:07:13.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:13.841854 7970 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.7.161.252:36527: Network error: Client connection negotiation failed: client connection to 127.7.161.252:36527: connect: Connection refused (error 111)
08:07:13.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:13.984779 7895 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.161.253" port: 44339 } attempt: 7
08:07:14.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:14.033342 7895 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.161.252" port: 36527 } has no permanent_uuid. Determining permanent_uuid...
08:07:14.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:14.048568 7895 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.7.161.252:36527: Network error: Client connection negotiation failed: client connection to 127.7.161.252:36527: connect: Connection refused (error 111)
08:07:14.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:14.107161 7895 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.161.252" port: 36527 } attempt: 1
08:07:14.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:14.115108 7895 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.7.161.252:36527: Network error: Client connection negotiation failed: client connection to 127.7.161.252:36527: connect: Connection refused (error 111)
08:07:14.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:14.141454 7970 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.161.252" port: 36527 } attempt: 5
08:07:14.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:14.149164 7970 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.7.161.252:36527: Network error: Client connection negotiation failed: client connection to 127.7.161.252:36527: connect: Connection refused (error 111)
08:07:14.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:14.196673 7895 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.161.252" port: 36527 } attempt: 2
08:07:14.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:14.204160 7895 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.7.161.252:36527: Network error: Client connection negotiation failed: client connection to 127.7.161.252:36527: connect: Connection refused (error 111)
08:07:14.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:14.289721 7895 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.161.252" port: 36527 } attempt: 3
08:07:14.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:14.296922 7895 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.7.161.252:36527: Network error: Client connection negotiation failed: client connection to 127.7.161.252:36527: connect: Connection refused (error 111)
08:07:14.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:14.437472 7895 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.161.252" port: 36527 } attempt: 4
08:07:14.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:14.445324 7895 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.7.161.252:36527: Network error: Client connection negotiation failed: client connection to 127.7.161.252:36527: connect: Connection refused (error 111)
08:07:14.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:14.696705 7970 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.161.252" port: 36527 } attempt: 6
08:07:14.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:14.728863 7895 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.161.252" port: 36527 } attempt: 5
08:07:14.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:14.736653 7895 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.7.161.252:36527: Network error: Client connection negotiation failed: client connection to 127.7.161.252:36527: connect: Connection refused (error 111)
08:07:14.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:14.738746 7970 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.7.161.252:36527: Network error: Client connection negotiation failed: client connection to 127.7.161.252:36527: connect: Connection refused (error 111)
08:07:15.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:15.289094 7895 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.161.252" port: 36527 } attempt: 6
08:07:15.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:15.214305 7981 debug-util.cc:398] Leaking SignalData structure 0x7b0800037520 after lost signal to thread 7972
08:07:15.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:15.317587 7981 kernel_stack_watchdog.cc:198] Thread 7972 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 400ms:
08:07:15.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
08:07:15.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
08:07:15.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
08:07:15.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
08:07:15.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:15.319010 7972 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.508s user 0.550s sys 0.951s
08:07:15.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:15.319381 7972 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.509s user 0.550s sys 0.951s
08:07:15.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:13.811182 7983 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:07:15.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:15.320216 7985 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:07:15.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:15.323765 7984 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1508 milliseconds
08:07:15.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:15.324283 7972 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
08:07:15.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:15.326114 7895 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.7.161.252:36527: Network error: Client connection negotiation failed: client connection to 127.7.161.252:36527: connect: Connection refused (error 111)
08:07:15.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:15.326622 7972 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:07:15.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:15.329555 7972 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:07:15.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:15.331238 7972 hybrid_clock.cc:648] HybridClock initialized: now 1761206835331185 us; error 35 us; skew 500 ppm
08:07:15.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:15.332325 7972 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:07:15.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:15.339123 7972 webserver.cc:492] Webserver started at http://127.7.161.252:35917/ using document root <none> and password file <none>
08:07:15.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:15.339998 7972 fs_manager.cc:362] Metadata directory not provided
08:07:15.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:15.340184 7972 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:07:15.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:15.340612 7972 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
08:07:15.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:15.344727 7972 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster3944160619542668700/master-2/data/instance:
08:07:15.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "56db3854fc23457fa53cc786ad1c8ff7"
08:07:15.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-23 08:07:15 on dist-test-slave-wxqn"
08:07:15.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:15.345768 7972 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster3944160619542668700/master-2/wal/instance:
08:07:15.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "56db3854fc23457fa53cc786ad1c8ff7"
08:07:15.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-23 08:07:15 on dist-test-slave-wxqn"
08:07:15.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:15.352698 7972 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.006s sys 0.002s
08:07:15.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:15.358204 7999 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:07:15.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:15.359268 7972 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
08:07:15.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:15.359555 7972 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster3944160619542668700/master-2/data,/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster3944160619542668700/master-2/wal
08:07:15.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "56db3854fc23457fa53cc786ad1c8ff7"
08:07:15.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-23 08:07:15 on dist-test-slave-wxqn"
08:07:15.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:15.359843 7972 fs_report.cc:389] FS layout report
08:07:15.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:07:15.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster3944160619542668700/master-2/wal
08:07:15.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster3944160619542668700/master-2/wal
08:07:15.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster3944160619542668700/master-2/data/data
08:07:15.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:07:15.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:07:15.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:07:15.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:07:15.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:07:15.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:07:15.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:07:15.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:07:15.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:07:15.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:07:15.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:07:15.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:15.413450 7972 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:07:15.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:15.414871 7972 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:07:15.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:15.415318 7972 kserver.cc:163] Server-wide thread pool size limit: 3276
08:07:15.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:15.484489 7972 rpc_server.cc:307] RPC server started. Bound to: 127.7.161.252:36527
08:07:15.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:15.484593 8050 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.7.161.252:36527 every 8 connection(s)
08:07:15.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:15.487028 7972 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster3944160619542668700/master-2/data/info.pb
08:07:15.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:15.492537 8051 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
08:07:15.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:15.494745 7815 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskcrgkbR/build/tsan/bin/kudu as pid 7972
08:07:15.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:15.495085 7815 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster3944160619542668700/master-2/wal/instance
08:07:15.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:15.507658 8051 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.161.254" port: 39651 } has no permanent_uuid. Determining permanent_uuid...
08:07:15.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:15.535470 8051 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.161.253" port: 44339 } has no permanent_uuid. Determining permanent_uuid...
08:07:15.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:15.544924 8051 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.161.252" port: 36527 } has no permanent_uuid. Determining permanent_uuid...
08:07:15.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:15.560122 8051 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 56db3854fc23457fa53cc786ad1c8ff7: Bootstrap starting.
08:07:15.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:15.565873 8051 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 56db3854fc23457fa53cc786ad1c8ff7: Neither blocks nor log segments found. Creating new log.
08:07:15.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:15.568065 8051 log.cc:826] T 00000000000000000000000000000000 P 56db3854fc23457fa53cc786ad1c8ff7: Log is configured to *not* fsync() on all Append() calls
08:07:15.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:15.572117 8051 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 56db3854fc23457fa53cc786ad1c8ff7: No bootstrap required, opened a new log
08:07:15.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:15.591136 8051 raft_consensus.cc:359] T 00000000000000000000000000000000 P 56db3854fc23457fa53cc786ad1c8ff7 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e08a5f066ba34c6f98c017e1304905bb" member_type: VOTER last_known_addr { host: "127.7.161.254" port: 39651 } } peers { permanent_uuid: "c0a6c76326f44dd4a06424b1bb421f76" member_type: VOTER last_known_addr { host: "127.7.161.253" port: 44339 } } peers { permanent_uuid: "56db3854fc23457fa53cc786ad1c8ff7" member_type: VOTER last_known_addr { host: "127.7.161.252" port: 36527 } }
08:07:15.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:15.591763 8051 raft_consensus.cc:385] T 00000000000000000000000000000000 P 56db3854fc23457fa53cc786ad1c8ff7 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:07:15.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:15.591946 8051 raft_consensus.cc:740] T 00000000000000000000000000000000 P 56db3854fc23457fa53cc786ad1c8ff7 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 56db3854fc23457fa53cc786ad1c8ff7, State: Initialized, Role: FOLLOWER
08:07:15.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:15.592653 8051 consensus_queue.cc:260] T 00000000000000000000000000000000 P 56db3854fc23457fa53cc786ad1c8ff7 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e08a5f066ba34c6f98c017e1304905bb" member_type: VOTER last_known_addr { host: "127.7.161.254" port: 39651 } } peers { permanent_uuid: "c0a6c76326f44dd4a06424b1bb421f76" member_type: VOTER last_known_addr { host: "127.7.161.253" port: 44339 } } peers { permanent_uuid: "56db3854fc23457fa53cc786ad1c8ff7" member_type: VOTER last_known_addr { host: "127.7.161.252" port: 36527 } }
08:07:15.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:15.595412 8060 sys_catalog.cc:455] T 00000000000000000000000000000000 P 56db3854fc23457fa53cc786ad1c8ff7 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e08a5f066ba34c6f98c017e1304905bb" member_type: VOTER last_known_addr { host: "127.7.161.254" port: 39651 } } peers { permanent_uuid: "c0a6c76326f44dd4a06424b1bb421f76" member_type: VOTER last_known_addr { host: "127.7.161.253" port: 44339 } } peers { permanent_uuid: "56db3854fc23457fa53cc786ad1c8ff7" member_type: VOTER last_known_addr { host: "127.7.161.252" port: 36527 } } }
08:07:15.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:15.596242 8060 sys_catalog.cc:458] T 00000000000000000000000000000000 P 56db3854fc23457fa53cc786ad1c8ff7 [sys.catalog]: This master's current role is: FOLLOWER
08:07:15.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:15.597265 8051 sys_catalog.cc:565] T 00000000000000000000000000000000 P 56db3854fc23457fa53cc786ad1c8ff7 [sys.catalog]: configured and running, proceeding with master startup.
08:07:15.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:15.620574 8071 catalog_manager.cc:1568] T 00000000000000000000000000000000 P 56db3854fc23457fa53cc786ad1c8ff7: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:07:15.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:15.621301 8071 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:07:15.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:15.617193 7972 master_runner.cc:186] Error getting master registration for 127.7.161.254:39651: OK, instance_id { permanent_uuid: "e08a5f066ba34c6f98c017e1304905bb" instance_seqno: 1761206831606746 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
08:07:15.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:15.641914 7972 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
08:07:15.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:15.670864 8005 tablet.cc:2378] T 00000000000000000000000000000000 P 56db3854fc23457fa53cc786ad1c8ff7: Can't schedule compaction. Clean time has not been advanced past its initial value.
08:07:15.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:15.799226 7970 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.161.252" port: 36527 } attempt: 7
08:07:15.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:15.817649 7970 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c0a6c76326f44dd4a06424b1bb421f76: Bootstrap starting.
08:07:15.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:15.823040 7970 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P c0a6c76326f44dd4a06424b1bb421f76: Neither blocks nor log segments found. Creating new log.
08:07:15.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:15.824838 7970 log.cc:826] T 00000000000000000000000000000000 P c0a6c76326f44dd4a06424b1bb421f76: Log is configured to *not* fsync() on all Append() calls
08:07:15.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:15.829136 7970 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c0a6c76326f44dd4a06424b1bb421f76: No bootstrap required, opened a new log
08:07:15.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:15.847399 7970 raft_consensus.cc:359] T 00000000000000000000000000000000 P c0a6c76326f44dd4a06424b1bb421f76 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e08a5f066ba34c6f98c017e1304905bb" member_type: VOTER last_known_addr { host: "127.7.161.254" port: 39651 } } peers { permanent_uuid: "c0a6c76326f44dd4a06424b1bb421f76" member_type: VOTER last_known_addr { host: "127.7.161.253" port: 44339 } } peers { permanent_uuid: "56db3854fc23457fa53cc786ad1c8ff7" member_type: VOTER last_known_addr { host: "127.7.161.252" port: 36527 } }
08:07:15.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:15.848058 7970 raft_consensus.cc:385] T 00000000000000000000000000000000 P c0a6c76326f44dd4a06424b1bb421f76 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:07:15.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:15.848253 7970 raft_consensus.cc:740] T 00000000000000000000000000000000 P c0a6c76326f44dd4a06424b1bb421f76 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c0a6c76326f44dd4a06424b1bb421f76, State: Initialized, Role: FOLLOWER
08:07:15.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:15.848992 7970 consensus_queue.cc:260] T 00000000000000000000000000000000 P c0a6c76326f44dd4a06424b1bb421f76 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e08a5f066ba34c6f98c017e1304905bb" member_type: VOTER last_known_addr { host: "127.7.161.254" port: 39651 } } peers { permanent_uuid: "c0a6c76326f44dd4a06424b1bb421f76" member_type: VOTER last_known_addr { host: "127.7.161.253" port: 44339 } } peers { permanent_uuid: "56db3854fc23457fa53cc786ad1c8ff7" member_type: VOTER last_known_addr { host: "127.7.161.252" port: 36527 } }
08:07:15.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:15.851349 8075 sys_catalog.cc:455] T 00000000000000000000000000000000 P c0a6c76326f44dd4a06424b1bb421f76 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e08a5f066ba34c6f98c017e1304905bb" member_type: VOTER last_known_addr { host: "127.7.161.254" port: 39651 } } peers { permanent_uuid: "c0a6c76326f44dd4a06424b1bb421f76" member_type: VOTER last_known_addr { host: "127.7.161.253" port: 44339 } } peers { permanent_uuid: "56db3854fc23457fa53cc786ad1c8ff7" member_type: VOTER last_known_addr { host: "127.7.161.252" port: 36527 } } }
08:07:15.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:15.852947 8075 sys_catalog.cc:458] T 00000000000000000000000000000000 P c0a6c76326f44dd4a06424b1bb421f76 [sys.catalog]: This master's current role is: FOLLOWER
08:07:15.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:15.853303 7970 sys_catalog.cc:565] T 00000000000000000000000000000000 P c0a6c76326f44dd4a06424b1bb421f76 [sys.catalog]: configured and running, proceeding with master startup.
08:07:15.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:15.884644 8086 catalog_manager.cc:1568] T 00000000000000000000000000000000 P c0a6c76326f44dd4a06424b1bb421f76: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:07:15.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:15.886433 8086 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:07:15.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:15.879760 7897 master_runner.cc:186] Error getting master registration for 127.7.161.254:39651: OK, instance_id { permanent_uuid: "e08a5f066ba34c6f98c017e1304905bb" instance_seqno: 1761206831606746 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
08:07:15.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:15.909974 7897 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
08:07:15.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:15.981909 8060 raft_consensus.cc:493] T 00000000000000000000000000000000 P 56db3854fc23457fa53cc786ad1c8ff7 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:07:15.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:15.982535 8060 raft_consensus.cc:515] T 00000000000000000000000000000000 P 56db3854fc23457fa53cc786ad1c8ff7 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e08a5f066ba34c6f98c017e1304905bb" member_type: VOTER last_known_addr { host: "127.7.161.254" port: 39651 } } peers { permanent_uuid: "c0a6c76326f44dd4a06424b1bb421f76" member_type: VOTER last_known_addr { host: "127.7.161.253" port: 44339 } } peers { permanent_uuid: "56db3854fc23457fa53cc786ad1c8ff7" member_type: VOTER last_known_addr { host: "127.7.161.252" port: 36527 } }
08:07:15.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:15.985119 8060 leader_election.cc:290] T 00000000000000000000000000000000 P 56db3854fc23457fa53cc786ad1c8ff7 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers e08a5f066ba34c6f98c017e1304905bb (127.7.161.254:39651), c0a6c76326f44dd4a06424b1bb421f76 (127.7.161.253:44339)
08:07:15.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:15.987540 7945 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "56db3854fc23457fa53cc786ad1c8ff7" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c0a6c76326f44dd4a06424b1bb421f76" is_pre_election: true
08:07:15.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:15.988462 7945 raft_consensus.cc:2468] T 00000000000000000000000000000000 P c0a6c76326f44dd4a06424b1bb421f76 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 56db3854fc23457fa53cc786ad1c8ff7 in term 0.
08:07:15.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:15.989902 8003 leader_election.cc:304] T 00000000000000000000000000000000 P 56db3854fc23457fa53cc786ad1c8ff7 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 56db3854fc23457fa53cc786ad1c8ff7, c0a6c76326f44dd4a06424b1bb421f76; no voters:
08:07:15.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:15.990837 8060 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 56db3854fc23457fa53cc786ad1c8ff7 [term 0 FOLLOWER]: Leader pre-election won for term 1
08:07:15.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:15.991194 8060 raft_consensus.cc:493] T 00000000000000000000000000000000 P 56db3854fc23457fa53cc786ad1c8ff7 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
08:07:15.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:15.991518 8060 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 56db3854fc23457fa53cc786ad1c8ff7 [term 0 FOLLOWER]: Advancing to term 1
08:07:16.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:16.000809 8060 raft_consensus.cc:515] T 00000000000000000000000000000000 P 56db3854fc23457fa53cc786ad1c8ff7 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e08a5f066ba34c6f98c017e1304905bb" member_type: VOTER last_known_addr { host: "127.7.161.254" port: 39651 } } peers { permanent_uuid: "c0a6c76326f44dd4a06424b1bb421f76" member_type: VOTER last_known_addr { host: "127.7.161.253" port: 44339 } } peers { permanent_uuid: "56db3854fc23457fa53cc786ad1c8ff7" member_type: VOTER last_known_addr { host: "127.7.161.252" port: 36527 } }
08:07:16.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:16.010268 8060 leader_election.cc:290] T 00000000000000000000000000000000 P 56db3854fc23457fa53cc786ad1c8ff7 [CANDIDATE]: Term 1 election: Requested vote from peers e08a5f066ba34c6f98c017e1304905bb (127.7.161.254:39651), c0a6c76326f44dd4a06424b1bb421f76 (127.7.161.253:44339)
08:07:16.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:16.010879 7945 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "56db3854fc23457fa53cc786ad1c8ff7" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c0a6c76326f44dd4a06424b1bb421f76"
08:07:16.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:16.011484 7945 raft_consensus.cc:3060] T 00000000000000000000000000000000 P c0a6c76326f44dd4a06424b1bb421f76 [term 0 FOLLOWER]: Advancing to term 1
08:07:16.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:16.018930 7945 raft_consensus.cc:2468] T 00000000000000000000000000000000 P c0a6c76326f44dd4a06424b1bb421f76 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 56db3854fc23457fa53cc786ad1c8ff7 in term 1.
08:07:16.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:16.020987 8003 leader_election.cc:304] T 00000000000000000000000000000000 P 56db3854fc23457fa53cc786ad1c8ff7 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 56db3854fc23457fa53cc786ad1c8ff7, c0a6c76326f44dd4a06424b1bb421f76; no voters:
08:07:16.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:16.021937 8060 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 56db3854fc23457fa53cc786ad1c8ff7 [term 1 FOLLOWER]: Leader election won for term 1
08:07:16.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:16.024142 8060 raft_consensus.cc:697] T 00000000000000000000000000000000 P 56db3854fc23457fa53cc786ad1c8ff7 [term 1 LEADER]: Becoming Leader. State: Replica: 56db3854fc23457fa53cc786ad1c8ff7, State: Running, Role: LEADER
08:07:16.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:16.025364 8060 consensus_queue.cc:237] T 00000000000000000000000000000000 P 56db3854fc23457fa53cc786ad1c8ff7 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e08a5f066ba34c6f98c017e1304905bb" member_type: VOTER last_known_addr { host: "127.7.161.254" port: 39651 } } peers { permanent_uuid: "c0a6c76326f44dd4a06424b1bb421f76" member_type: VOTER last_known_addr { host: "127.7.161.253" port: 44339 } } peers { permanent_uuid: "56db3854fc23457fa53cc786ad1c8ff7" member_type: VOTER last_known_addr { host: "127.7.161.252" port: 36527 } }
08:07:16.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:16.033423 8088 sys_catalog.cc:455] T 00000000000000000000000000000000 P 56db3854fc23457fa53cc786ad1c8ff7 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 56db3854fc23457fa53cc786ad1c8ff7. Latest consensus state: current_term: 1 leader_uuid: "56db3854fc23457fa53cc786ad1c8ff7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e08a5f066ba34c6f98c017e1304905bb" member_type: VOTER last_known_addr { host: "127.7.161.254" port: 39651 } } peers { permanent_uuid: "c0a6c76326f44dd4a06424b1bb421f76" member_type: VOTER last_known_addr { host: "127.7.161.253" port: 44339 } } peers { permanent_uuid: "56db3854fc23457fa53cc786ad1c8ff7" member_type: VOTER last_known_addr { host: "127.7.161.252" port: 36527 } } }
08:07:16.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:16.034288 8088 sys_catalog.cc:458] T 00000000000000000000000000000000 P 56db3854fc23457fa53cc786ad1c8ff7 [sys.catalog]: This master's current role is: LEADER
08:07:16.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:16.047109 8090 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
08:07:16.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:16.054833 8090 catalog_manager.cc:1494] Initializing Kudu cluster ID...
08:07:16.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:16.071384 7945 raft_consensus.cc:1275] T 00000000000000000000000000000000 P c0a6c76326f44dd4a06424b1bb421f76 [term 1 FOLLOWER]: Refusing update from remote peer 56db3854fc23457fa53cc786ad1c8ff7: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
08:07:16.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:16.077795 8088 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 56db3854fc23457fa53cc786ad1c8ff7 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c0a6c76326f44dd4a06424b1bb421f76" member_type: VOTER last_known_addr { host: "127.7.161.253" port: 44339 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
08:07:16.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:16.087648 7924 tablet.cc:2378] T 00000000000000000000000000000000 P c0a6c76326f44dd4a06424b1bb421f76: Can't schedule compaction. Clean time has not been advanced past its initial value.
08:07:16.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:16.107255 8091 mvcc.cc:204] Tried to move back new op lower bound from 7213903200532615168 to 7213903200370307072. Current Snapshot: MvccSnapshot[applied={T|T < 7213903200532615168}]
08:07:16.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:16.112571 8060 sys_catalog.cc:455] T 00000000000000000000000000000000 P 56db3854fc23457fa53cc786ad1c8ff7 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "56db3854fc23457fa53cc786ad1c8ff7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e08a5f066ba34c6f98c017e1304905bb" member_type: VOTER last_known_addr { host: "127.7.161.254" port: 39651 } } peers { permanent_uuid: "c0a6c76326f44dd4a06424b1bb421f76" member_type: VOTER last_known_addr { host: "127.7.161.253" port: 44339 } } peers { permanent_uuid: "56db3854fc23457fa53cc786ad1c8ff7" member_type: VOTER last_known_addr { host: "127.7.161.252" port: 36527 } } }
08:07:16.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:16.113262 8060 sys_catalog.cc:458] T 00000000000000000000000000000000 P 56db3854fc23457fa53cc786ad1c8ff7 [sys.catalog]: This master's current role is: LEADER
08:07:16.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:16.116984 8090 catalog_manager.cc:1357] Generated new cluster ID: ce0eb0df83ae4cb4abc3504a492fb80d
08:07:16.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:16.117275 8090 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
08:07:16.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:16.128563 8075 sys_catalog.cc:455] T 00000000000000000000000000000000 P c0a6c76326f44dd4a06424b1bb421f76 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 56db3854fc23457fa53cc786ad1c8ff7. Latest consensus state: current_term: 1 leader_uuid: "56db3854fc23457fa53cc786ad1c8ff7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e08a5f066ba34c6f98c017e1304905bb" member_type: VOTER last_known_addr { host: "127.7.161.254" port: 39651 } } peers { permanent_uuid: "c0a6c76326f44dd4a06424b1bb421f76" member_type: VOTER last_known_addr { host: "127.7.161.253" port: 44339 } } peers { permanent_uuid: "56db3854fc23457fa53cc786ad1c8ff7" member_type: VOTER last_known_addr { host: "127.7.161.252" port: 36527 } } }
08:07:16.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:16.138631 8075 sys_catalog.cc:458] T 00000000000000000000000000000000 P c0a6c76326f44dd4a06424b1bb421f76 [sys.catalog]: This master's current role is: FOLLOWER
08:07:16.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:16.146800 8095 sys_catalog.cc:455] T 00000000000000000000000000000000 P c0a6c76326f44dd4a06424b1bb421f76 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "56db3854fc23457fa53cc786ad1c8ff7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e08a5f066ba34c6f98c017e1304905bb" member_type: VOTER last_known_addr { host: "127.7.161.254" port: 39651 } } peers { permanent_uuid: "c0a6c76326f44dd4a06424b1bb421f76" member_type: VOTER last_known_addr { host: "127.7.161.253" port: 44339 } } peers { permanent_uuid: "56db3854fc23457fa53cc786ad1c8ff7" member_type: VOTER last_known_addr { host: "127.7.161.252" port: 36527 } } }
08:07:16.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:16.147547 8095 sys_catalog.cc:458] T 00000000000000000000000000000000 P c0a6c76326f44dd4a06424b1bb421f76 [sys.catalog]: This master's current role is: FOLLOWER
08:07:16.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:16.166886 8090 catalog_manager.cc:1380] Generated new certificate authority record
08:07:16.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:16.170430 8090 catalog_manager.cc:1514] Loading token signing keys...
08:07:16.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:16.204277 8090 catalog_manager.cc:6022] T 00000000000000000000000000000000 P 56db3854fc23457fa53cc786ad1c8ff7: Generated new TSK 0
08:07:16.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:16.205569 8090 catalog_manager.cc:1524] Initializing in-progress tserver states...
08:07:16.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:16.231248 7815 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskcrgkbR/build/tsan/bin/kudu
08:07:16.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskcrgkbR/build/tsan/bin/kudu
08:07:16.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster3944160619542668700/ts-0/wal
08:07:16.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster3944160619542668700/ts-0/data
08:07:16.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:07:16.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:07:16.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:07:16.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:07:16.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:07:16.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:07:16.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster3944160619542668700/ts-0/logs
08:07:16.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster3944160619542668700/ts-0/data/info.pb
08:07:16.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:07:16.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:07:16.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:07:16.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:07:16.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:07:16.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:07:16.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:07:16.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:07:16.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
08:07:16.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:07:16.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.161.193:0
08:07:16.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.7.161.193
08:07:16.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.161.193
08:07:16.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:07:16.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.7.161.254:39651,127.7.161.253:44339,127.7.161.252:36527
08:07:16.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.161.212:38263
08:07:16.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:07:16.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:07:16.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
08:07:16.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:16.359589 7895 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.161.252" port: 36527 } attempt: 7
08:07:16.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:16.429575 7895 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P e08a5f066ba34c6f98c017e1304905bb: Bootstrap starting.
08:07:16.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:16.447940 7895 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P e08a5f066ba34c6f98c017e1304905bb: Neither blocks nor log segments found. Creating new log.
08:07:16.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:16.450683 7895 log.cc:826] T 00000000000000000000000000000000 P e08a5f066ba34c6f98c017e1304905bb: Log is configured to *not* fsync() on all Append() calls
08:07:16.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:16.469878 7895 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P e08a5f066ba34c6f98c017e1304905bb: No bootstrap required, opened a new log
08:07:16.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:16.523313 7895 raft_consensus.cc:359] T 00000000000000000000000000000000 P e08a5f066ba34c6f98c017e1304905bb [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e08a5f066ba34c6f98c017e1304905bb" member_type: VOTER last_known_addr { host: "127.7.161.254" port: 39651 } } peers { permanent_uuid: "c0a6c76326f44dd4a06424b1bb421f76" member_type: VOTER last_known_addr { host: "127.7.161.253" port: 44339 } } peers { permanent_uuid: "56db3854fc23457fa53cc786ad1c8ff7" member_type: VOTER last_known_addr { host: "127.7.161.252" port: 36527 } }
08:07:16.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:16.524621 7895 raft_consensus.cc:385] T 00000000000000000000000000000000 P e08a5f066ba34c6f98c017e1304905bb [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:07:16.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:16.525799 7895 raft_consensus.cc:740] T 00000000000000000000000000000000 P e08a5f066ba34c6f98c017e1304905bb [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e08a5f066ba34c6f98c017e1304905bb, State: Initialized, Role: FOLLOWER
08:07:16.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:16.527078 7895 consensus_queue.cc:260] T 00000000000000000000000000000000 P e08a5f066ba34c6f98c017e1304905bb [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e08a5f066ba34c6f98c017e1304905bb" member_type: VOTER last_known_addr { host: "127.7.161.254" port: 39651 } } peers { permanent_uuid: "c0a6c76326f44dd4a06424b1bb421f76" member_type: VOTER last_known_addr { host: "127.7.161.253" port: 44339 } } peers { permanent_uuid: "56db3854fc23457fa53cc786ad1c8ff7" member_type: VOTER last_known_addr { host: "127.7.161.252" port: 36527 } }
08:07:16.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:16.541514 8101 sys_catalog.cc:455] T 00000000000000000000000000000000 P e08a5f066ba34c6f98c017e1304905bb [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e08a5f066ba34c6f98c017e1304905bb" member_type: VOTER last_known_addr { host: "127.7.161.254" port: 39651 } } peers { permanent_uuid: "c0a6c76326f44dd4a06424b1bb421f76" member_type: VOTER last_known_addr { host: "127.7.161.253" port: 44339 } } peers { permanent_uuid: "56db3854fc23457fa53cc786ad1c8ff7" member_type: VOTER last_known_addr { host: "127.7.161.252" port: 36527 } } }
08:07:16.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:16.542594 8101 sys_catalog.cc:458] T 00000000000000000000000000000000 P e08a5f066ba34c6f98c017e1304905bb [sys.catalog]: This master's current role is: FOLLOWER
08:07:16.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:16.543396 7895 sys_catalog.cc:565] T 00000000000000000000000000000000 P e08a5f066ba34c6f98c017e1304905bb [sys.catalog]: configured and running, proceeding with master startup.
08:07:16.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:16.544111 7869 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "56db3854fc23457fa53cc786ad1c8ff7" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e08a5f066ba34c6f98c017e1304905bb"
08:07:16.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:16.544898 7869 raft_consensus.cc:3060] T 00000000000000000000000000000000 P e08a5f066ba34c6f98c017e1304905bb [term 0 FOLLOWER]: Advancing to term 1
08:07:16.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:16.548722 7870 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "56db3854fc23457fa53cc786ad1c8ff7" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e08a5f066ba34c6f98c017e1304905bb" is_pre_election: true
08:07:16.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:16.554389 7869 raft_consensus.cc:2468] T 00000000000000000000000000000000 P e08a5f066ba34c6f98c017e1304905bb [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 56db3854fc23457fa53cc786ad1c8ff7 in term 1.
08:07:16.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:16.559633 7868 raft_consensus.cc:1275] T 00000000000000000000000000000000 P e08a5f066ba34c6f98c017e1304905bb [term 1 FOLLOWER]: Refusing update from remote peer 56db3854fc23457fa53cc786ad1c8ff7: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
08:07:16.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:16.564810 7849 tablet.cc:2378] T 00000000000000000000000000000000 P e08a5f066ba34c6f98c017e1304905bb: Can't schedule compaction. Clean time has not been advanced past its initial value.
08:07:16.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:16.568329 8060 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 56db3854fc23457fa53cc786ad1c8ff7 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e08a5f066ba34c6f98c017e1304905bb" member_type: VOTER last_known_addr { host: "127.7.161.254" port: 39651 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
08:07:16.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:16.661928 8101 sys_catalog.cc:455] T 00000000000000000000000000000000 P e08a5f066ba34c6f98c017e1304905bb [sys.catalog]: SysCatalogTable state changed. Reason: New leader 56db3854fc23457fa53cc786ad1c8ff7. Latest consensus state: current_term: 1 leader_uuid: "56db3854fc23457fa53cc786ad1c8ff7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e08a5f066ba34c6f98c017e1304905bb" member_type: VOTER last_known_addr { host: "127.7.161.254" port: 39651 } } peers { permanent_uuid: "c0a6c76326f44dd4a06424b1bb421f76" member_type: VOTER last_known_addr { host: "127.7.161.253" port: 44339 } } peers { permanent_uuid: "56db3854fc23457fa53cc786ad1c8ff7" member_type: VOTER last_known_addr { host: "127.7.161.252" port: 36527 } } }
08:07:16.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:16.662645 8101 sys_catalog.cc:458] T 00000000000000000000000000000000 P e08a5f066ba34c6f98c017e1304905bb [sys.catalog]: This master's current role is: FOLLOWER
08:07:16.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:16.664862 8108 sys_catalog.cc:455] T 00000000000000000000000000000000 P e08a5f066ba34c6f98c017e1304905bb [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "56db3854fc23457fa53cc786ad1c8ff7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e08a5f066ba34c6f98c017e1304905bb" member_type: VOTER last_known_addr { host: "127.7.161.254" port: 39651 } } peers { permanent_uuid: "c0a6c76326f44dd4a06424b1bb421f76" member_type: VOTER last_known_addr { host: "127.7.161.253" port: 44339 } } peers { permanent_uuid: "56db3854fc23457fa53cc786ad1c8ff7" member_type: VOTER last_known_addr { host: "127.7.161.252" port: 36527 } } }
08:07:16.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:16.666363 8108 sys_catalog.cc:458] T 00000000000000000000000000000000 P e08a5f066ba34c6f98c017e1304905bb [sys.catalog]: This master's current role is: FOLLOWER
08:07:16.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:16.720849 8118 catalog_manager.cc:1269] Loaded cluster ID: ce0eb0df83ae4cb4abc3504a492fb80d
08:07:16.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:16.725871 8118 catalog_manager.cc:1562] T 00000000000000000000000000000000 P e08a5f066ba34c6f98c017e1304905bb: loading cluster ID for follower catalog manager: success
08:07:16.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:16.733654 8118 catalog_manager.cc:1584] T 00000000000000000000000000000000 P e08a5f066ba34c6f98c017e1304905bb: acquiring CA information for follower catalog manager: success
08:07:16.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:16.751641 8118 catalog_manager.cc:1612] T 00000000000000000000000000000000 P e08a5f066ba34c6f98c017e1304905bb: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
08:07:16.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:16.892774 8086 catalog_manager.cc:1269] Loaded cluster ID: ce0eb0df83ae4cb4abc3504a492fb80d
08:07:16.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:16.893932 8086 catalog_manager.cc:1562] T 00000000000000000000000000000000 P c0a6c76326f44dd4a06424b1bb421f76: loading cluster ID for follower catalog manager: success
08:07:16.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:16.903119 8097 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:07:16.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:16.907439 8086 catalog_manager.cc:1584] T 00000000000000000000000000000000 P c0a6c76326f44dd4a06424b1bb421f76: acquiring CA information for follower catalog manager: success
08:07:16.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:16.912178 8097 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:07:16.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:16.914281 8097 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:07:16.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:16.923699 8086 catalog_manager.cc:1612] T 00000000000000000000000000000000 P c0a6c76326f44dd4a06424b1bb421f76: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
08:07:16.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:16.987676 8097 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:07:16.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:16.989357 8097 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.7.161.193
08:07:17.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:17.053922 8097 tablet_server_runner.cc:78] Tablet server non-default flags:
08:07:17.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:07:17.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.161.212:38263
08:07:17.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:07:17.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:07:17.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster3944160619542668700/ts-0/data
08:07:17.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster3944160619542668700/ts-0/wal
08:07:17.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:07:17.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:07:17.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.161.193:0
08:07:17.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:07:17.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:07:17.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:07:17.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster3944160619542668700/ts-0/data/info.pb
08:07:17.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.161.193
08:07:17.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:07:17.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.7.161.254:39651,127.7.161.253:44339,127.7.161.252:36527
08:07:17.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:07:17.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:07:17.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:07:17.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:07:17.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:07:17.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.7.161.193
08:07:17.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster3944160619542668700/ts-0/logs
08:07:17.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:07:17.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:07:17.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:07:17.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
08:07:17.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:07:17.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 18b1930e830609248ee2e26d6216d073ff40ac5d
08:07:17.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:07:17.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 23 Oct 2025 07:43:21 UTC on 5fd53c4cbb9d
08:07:17.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8677
08:07:17.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:07:17.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:17.062357 8097 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:07:17.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:17.065366 8097 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:07:17.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:17.105882 8125 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:07:17.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:17.107053 8060 sys_catalog.cc:455] T 00000000000000000000000000000000 P 56db3854fc23457fa53cc786ad1c8ff7 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "56db3854fc23457fa53cc786ad1c8ff7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e08a5f066ba34c6f98c017e1304905bb" member_type: VOTER last_known_addr { host: "127.7.161.254" port: 39651 } } peers { permanent_uuid: "c0a6c76326f44dd4a06424b1bb421f76" member_type: VOTER last_known_addr { host: "127.7.161.253" port: 44339 } } peers { permanent_uuid: "56db3854fc23457fa53cc786ad1c8ff7" member_type: VOTER last_known_addr { host: "127.7.161.252" port: 36527 } } }
08:07:17.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:17.108649 8060 sys_catalog.cc:458] T 00000000000000000000000000000000 P 56db3854fc23457fa53cc786ad1c8ff7 [sys.catalog]: This master's current role is: LEADER
08:07:19.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:18.490785 8124 debug-util.cc:398] Leaking SignalData structure 0x7b0800037500 after lost signal to thread 8097
08:07:19.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:19.562176 8097 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 2.453s user 0.733s sys 1.023s
08:07:19.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:19.562584 8097 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 2.454s user 0.733s sys 1.023s
08:07:19.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:17.112742 8126 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:07:19.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:19.572019 8097 server_base.cc:1047] running on GCE node
08:07:19.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:19.573155 8133 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:07:19.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:19.574429 8097 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:07:19.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:19.576875 8097 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:07:19.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:19.578339 8097 hybrid_clock.cc:648] HybridClock initialized: now 1761206839578287 us; error 54 us; skew 500 ppm
08:07:19.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:19.579459 8097 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:07:19.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:19.590948 8097 webserver.cc:492] Webserver started at http://127.7.161.193:34939/ using document root <none> and password file <none>
08:07:19.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:19.592051 8097 fs_manager.cc:362] Metadata directory not provided
08:07:19.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:19.592298 8097 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:07:19.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:19.592820 8097 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
08:07:19.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:19.597065 8097 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster3944160619542668700/ts-0/data/instance:
08:07:19.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "965ebc5e8aca444099e620b581fccc2a"
08:07:19.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-23 08:07:19 on dist-test-slave-wxqn"
08:07:19.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:19.598373 8097 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster3944160619542668700/ts-0/wal/instance:
08:07:19.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "965ebc5e8aca444099e620b581fccc2a"
08:07:19.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-23 08:07:19 on dist-test-slave-wxqn"
08:07:19.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:19.606290 8097 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.004s sys 0.004s
08:07:19.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:19.612257 8140 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:07:19.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:19.613615 8097 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.001s
08:07:19.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:19.614040 8097 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster3944160619542668700/ts-0/data,/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster3944160619542668700/ts-0/wal
08:07:19.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "965ebc5e8aca444099e620b581fccc2a"
08:07:19.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-23 08:07:19 on dist-test-slave-wxqn"
08:07:19.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:19.614399 8097 fs_report.cc:389] FS layout report
08:07:19.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:07:19.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster3944160619542668700/ts-0/wal
08:07:19.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster3944160619542668700/ts-0/wal
08:07:19.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster3944160619542668700/ts-0/data/data
08:07:19.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:07:19.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:07:19.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:07:19.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:07:19.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:07:19.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:07:19.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:07:19.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:07:19.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:07:19.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:07:19.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:07:19.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:19.678591 8097 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:07:19.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:19.680121 8097 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:07:19.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:19.680670 8097 kserver.cc:163] Server-wide thread pool size limit: 3276
08:07:19.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:19.683956 8097 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
08:07:19.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:19.689043 8097 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
08:07:19.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:19.689267 8097 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
08:07:19.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:19.689564 8097 ts_tablet_manager.cc:616] Registered 0 tablets
08:07:19.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:19.689744 8097 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
08:07:19.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:19.851619 8097 rpc_server.cc:307] RPC server started. Bound to: 127.7.161.193:33937
08:07:19.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:19.851712 8252 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.7.161.193:33937 every 8 connection(s)
08:07:19.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:19.854583 8097 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster3944160619542668700/ts-0/data/info.pb
08:07:19.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:19.855554 7815 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskcrgkbR/build/tsan/bin/kudu as pid 8097
08:07:19.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:19.856191 7815 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster3944160619542668700/ts-0/wal/instance
08:07:19.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:19.870870 7815 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskcrgkbR/build/tsan/bin/kudu
08:07:19.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskcrgkbR/build/tsan/bin/kudu
08:07:19.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster3944160619542668700/ts-1/wal
08:07:19.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster3944160619542668700/ts-1/data
08:07:19.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:07:19.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:07:19.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:07:19.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:07:19.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:07:19.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:07:19.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster3944160619542668700/ts-1/logs
08:07:19.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster3944160619542668700/ts-1/data/info.pb
08:07:19.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:07:19.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:07:19.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:07:19.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:07:19.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:07:19.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:07:19.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:07:19.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:07:19.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
08:07:19.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:07:19.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.161.194:0
08:07:19.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.7.161.194
08:07:19.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.161.194
08:07:19.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:07:19.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.7.161.254:39651,127.7.161.253:44339,127.7.161.252:36527
08:07:19.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.161.212:38263
08:07:19.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:07:19.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:07:19.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
08:07:19.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:19.897053 8254 heartbeater.cc:344] Connected to a master server at 127.7.161.253:44339
08:07:19.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:19.897488 8254 heartbeater.cc:461] Registering TS with master...
08:07:19.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:19.898792 8254 heartbeater.cc:507] Master 127.7.161.253:44339 requested a full tablet report, sending...
08:07:19.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:19.901820 8255 heartbeater.cc:344] Connected to a master server at 127.7.161.254:39651
08:07:19.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:19.902196 8255 heartbeater.cc:461] Registering TS with master...
08:07:19.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:19.902199 7935 ts_manager.cc:194] Registered new tserver with Master: 965ebc5e8aca444099e620b581fccc2a (127.7.161.193:33937)
08:07:19.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:19.902554 8253 heartbeater.cc:344] Connected to a master server at 127.7.161.252:36527
08:07:19.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:19.902935 8253 heartbeater.cc:461] Registering TS with master...
08:07:19.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:19.903242 8255 heartbeater.cc:507] Master 127.7.161.254:39651 requested a full tablet report, sending...
08:07:19.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:19.904470 8253 heartbeater.cc:507] Master 127.7.161.252:36527 requested a full tablet report, sending...
08:07:19.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:19.907196 7860 ts_manager.cc:194] Registered new tserver with Master: 965ebc5e8aca444099e620b581fccc2a (127.7.161.193:33937)
08:07:19.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:19.907358 8016 ts_manager.cc:194] Registered new tserver with Master: 965ebc5e8aca444099e620b581fccc2a (127.7.161.193:33937)
08:07:19.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:19.909931 8016 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.7.161.193:46625
08:07:20.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:20.205312 8263 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:07:20.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:20.205789 8263 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:07:20.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:20.206211 8263 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:07:20.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:20.238209 8263 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:07:20.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:20.238981 8263 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.7.161.194
08:07:20.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:20.274353 8263 tablet_server_runner.cc:78] Tablet server non-default flags:
08:07:20.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:07:20.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.161.212:38263
08:07:20.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:07:20.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:07:20.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster3944160619542668700/ts-1/data
08:07:20.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster3944160619542668700/ts-1/wal
08:07:20.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:07:20.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:07:20.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.161.194:0
08:07:20.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:07:20.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:07:20.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:07:20.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster3944160619542668700/ts-1/data/info.pb
08:07:20.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.161.194
08:07:20.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:07:20.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.7.161.254:39651,127.7.161.253:44339,127.7.161.252:36527
08:07:20.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:07:20.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:07:20.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:07:20.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:07:20.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:07:20.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.7.161.194
08:07:20.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster3944160619542668700/ts-1/logs
08:07:20.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:07:20.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:07:20.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:07:20.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
08:07:20.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:07:20.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 18b1930e830609248ee2e26d6216d073ff40ac5d
08:07:20.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:07:20.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 23 Oct 2025 07:43:21 UTC on 5fd53c4cbb9d
08:07:20.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8677
08:07:20.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:07:20.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:20.275667 8263 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:07:20.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:20.277223 8263 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:07:20.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:20.291764 8269 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:07:20.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:20.913470 8253 heartbeater.cc:499] Master 127.7.161.252:36527 was elected leader, sending a full tablet report...
08:07:21.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:20.292814 8270 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:07:21.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:21.663075 8272 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:07:21.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:21.668808 8271 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1369 milliseconds
08:07:21.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:21.672766 8263 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.381s user 0.481s sys 0.869s
08:07:21.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:21.673424 8263 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.382s user 0.482s sys 0.869s
08:07:21.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:21.673748 8263 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
08:07:21.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:21.675273 8263 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:07:21.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:21.678303 8263 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:07:21.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:21.679883 8263 hybrid_clock.cc:648] HybridClock initialized: now 1761206841679823 us; error 67 us; skew 500 ppm
08:07:21.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:21.681121 8263 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:07:21.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:21.692219 8263 webserver.cc:492] Webserver started at http://127.7.161.194:44143/ using document root <none> and password file <none>
08:07:21.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:21.693702 8263 fs_manager.cc:362] Metadata directory not provided
08:07:21.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:21.693970 8263 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:07:21.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:21.694540 8263 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
08:07:21.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:21.701660 8263 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster3944160619542668700/ts-1/data/instance:
08:07:21.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c5138c71cecf48c8829f2ab18d05e251"
08:07:21.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-23 08:07:21 on dist-test-slave-wxqn"
08:07:21.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:21.703250 8263 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster3944160619542668700/ts-1/wal/instance:
08:07:21.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c5138c71cecf48c8829f2ab18d05e251"
08:07:21.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-23 08:07:21 on dist-test-slave-wxqn"
08:07:21.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:21.714128 8263 fs_manager.cc:696] Time spent creating directory manager: real 0.010s user 0.008s sys 0.004s
08:07:21.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:21.722451 8279 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:07:21.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:21.723850 8263 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.002s sys 0.004s
08:07:21.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:21.724267 8263 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster3944160619542668700/ts-1/data,/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster3944160619542668700/ts-1/wal
08:07:21.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c5138c71cecf48c8829f2ab18d05e251"
08:07:21.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-23 08:07:21 on dist-test-slave-wxqn"
08:07:21.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:21.724773 8263 fs_report.cc:389] FS layout report
08:07:21.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:07:21.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster3944160619542668700/ts-1/wal
08:07:21.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster3944160619542668700/ts-1/wal
08:07:21.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster3944160619542668700/ts-1/data/data
08:07:21.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:07:21.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:07:21.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:07:21.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:07:21.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:07:21.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:07:21.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:07:21.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:07:21.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:07:21.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:07:21.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:07:21.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:21.819074 8263 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:07:21.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:21.820649 8263 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:07:21.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:21.821116 8263 kserver.cc:163] Server-wide thread pool size limit: 3276
08:07:21.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:21.823644 8263 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
08:07:21.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:21.828290 8263 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
08:07:21.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:21.828536 8263 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
08:07:21.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:21.828833 8263 ts_tablet_manager.cc:616] Registered 0 tablets
08:07:21.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:21.829003 8263 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
08:07:21.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:21.996551 8263 rpc_server.cc:307] RPC server started. Bound to: 127.7.161.194:33697
08:07:21.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:21.996691 8392 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.7.161.194:33697 every 8 connection(s)
08:07:21.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:21.999287 8263 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster3944160619542668700/ts-1/data/info.pb
08:07:22.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:22.008549 7815 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskcrgkbR/build/tsan/bin/kudu as pid 8263
08:07:22.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:22.009147 7815 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster3944160619542668700/ts-1/wal/instance
08:07:22.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:22.025601 7815 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskcrgkbR/build/tsan/bin/kudu
08:07:22.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskcrgkbR/build/tsan/bin/kudu
08:07:22.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster3944160619542668700/ts-2/wal
08:07:22.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster3944160619542668700/ts-2/data
08:07:22.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:07:22.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:07:22.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:07:22.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:07:22.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:07:22.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:07:22.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster3944160619542668700/ts-2/logs
08:07:22.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster3944160619542668700/ts-2/data/info.pb
08:07:22.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:07:22.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:07:22.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:07:22.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:07:22.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:07:22.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:07:22.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:07:22.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:07:22.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
08:07:22.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:07:22.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.161.195:0
08:07:22.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.7.161.195
08:07:22.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.161.195
08:07:22.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:07:22.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.7.161.254:39651,127.7.161.253:44339,127.7.161.252:36527
08:07:22.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.161.212:38263
08:07:22.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:07:22.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:07:22.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
08:07:22.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:22.037073 8393 heartbeater.cc:344] Connected to a master server at 127.7.161.252:36527
08:07:22.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:22.037599 8393 heartbeater.cc:461] Registering TS with master...
08:07:22.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:22.038820 8393 heartbeater.cc:507] Master 127.7.161.252:36527 requested a full tablet report, sending...
08:07:22.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:22.041467 8016 ts_manager.cc:194] Registered new tserver with Master: c5138c71cecf48c8829f2ab18d05e251 (127.7.161.194:33697)
08:07:22.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:22.043170 8016 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.7.161.194:55015
08:07:22.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:22.050585 8394 heartbeater.cc:344] Connected to a master server at 127.7.161.253:44339
08:07:22.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:22.050927 8394 heartbeater.cc:461] Registering TS with master...
08:07:22.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:22.051612 8394 heartbeater.cc:507] Master 127.7.161.253:44339 requested a full tablet report, sending...
08:07:22.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:22.054257 8395 heartbeater.cc:344] Connected to a master server at 127.7.161.254:39651
08:07:22.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:22.054576 8395 heartbeater.cc:461] Registering TS with master...
08:07:22.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:22.054850 7935 ts_manager.cc:194] Registered new tserver with Master: c5138c71cecf48c8829f2ab18d05e251 (127.7.161.194:33697)
08:07:22.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:22.055131 8395 heartbeater.cc:507] Master 127.7.161.254:39651 requested a full tablet report, sending...
08:07:22.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:22.057588 7859 ts_manager.cc:194] Registered new tserver with Master: c5138c71cecf48c8829f2ab18d05e251 (127.7.161.194:33697)
08:07:22.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:22.368460 8401 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:07:22.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:22.368973 8401 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:07:22.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:22.369411 8401 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:07:22.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:22.402117 8401 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:07:22.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:22.402964 8401 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.7.161.195
08:07:22.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:22.439368 8401 tablet_server_runner.cc:78] Tablet server non-default flags:
08:07:22.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:07:22.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.161.212:38263
08:07:22.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:07:22.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:07:22.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster3944160619542668700/ts-2/data
08:07:22.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster3944160619542668700/ts-2/wal
08:07:22.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:07:22.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:07:22.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.161.195:0
08:07:22.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:07:22.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:07:22.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:07:22.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster3944160619542668700/ts-2/data/info.pb
08:07:22.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.161.195
08:07:22.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:07:22.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.7.161.254:39651,127.7.161.253:44339,127.7.161.252:36527
08:07:22.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:07:22.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:07:22.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:07:22.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:07:22.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:07:22.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.7.161.195
08:07:22.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster3944160619542668700/ts-2/logs
08:07:22.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:07:22.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:07:22.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:07:22.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
08:07:22.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:07:22.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 18b1930e830609248ee2e26d6216d073ff40ac5d
08:07:22.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:07:22.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 23 Oct 2025 07:43:21 UTC on 5fd53c4cbb9d
08:07:22.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8677
08:07:22.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:07:22.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:22.440873 8401 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:07:22.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:22.442590 8401 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:07:22.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:22.456010 8408 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:07:23.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:23.048378 8393 heartbeater.cc:499] Master 127.7.161.252:36527 was elected leader, sending a full tablet report...
08:07:23.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:22.458055 8409 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:07:23.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:23.725106 8411 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:07:23.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:23.727236 8410 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1265 milliseconds
08:07:23.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:23.727380 8401 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
08:07:23.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:23.728658 8401 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:07:23.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:23.732604 8401 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:07:23.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:23.733948 8401 hybrid_clock.cc:648] HybridClock initialized: now 1761206843733913 us; error 42 us; skew 500 ppm
08:07:23.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:23.734722 8401 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:07:23.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:23.740589 8401 webserver.cc:492] Webserver started at http://127.7.161.195:34755/ using document root <none> and password file <none>
08:07:23.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:23.741461 8401 fs_manager.cc:362] Metadata directory not provided
08:07:23.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:23.741650 8401 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:07:23.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:23.742040 8401 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
08:07:23.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:23.746186 8401 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster3944160619542668700/ts-2/data/instance:
08:07:23.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f34c4dd855bb4d69a094f8670a4b9a46"
08:07:23.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-23 08:07:23 on dist-test-slave-wxqn"
08:07:23.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:23.747196 8401 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster3944160619542668700/ts-2/wal/instance:
08:07:23.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f34c4dd855bb4d69a094f8670a4b9a46"
08:07:23.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-23 08:07:23 on dist-test-slave-wxqn"
08:07:23.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:23.754077 8401 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.006s sys 0.003s
08:07:23.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:23.759871 8419 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:07:23.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:23.760957 8401 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
08:07:23.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:23.761247 8401 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster3944160619542668700/ts-2/data,/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster3944160619542668700/ts-2/wal
08:07:23.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f34c4dd855bb4d69a094f8670a4b9a46"
08:07:23.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-23 08:07:23 on dist-test-slave-wxqn"
08:07:23.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:23.761533 8401 fs_report.cc:389] FS layout report
08:07:23.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:07:23.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster3944160619542668700/ts-2/wal
08:07:23.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster3944160619542668700/ts-2/wal
08:07:23.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster3944160619542668700/ts-2/data/data
08:07:23.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:07:23.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:07:23.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:07:23.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:07:23.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:07:23.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:07:23.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:07:23.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:07:23.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:07:23.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:07:23.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:07:23.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:23.812043 8401 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:07:23.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:23.813532 8401 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:07:23.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:23.813934 8401 kserver.cc:163] Server-wide thread pool size limit: 3276
08:07:23.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:23.816377 8401 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
08:07:23.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:23.820297 8401 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
08:07:23.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:23.820477 8401 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
08:07:23.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:23.820726 8401 ts_tablet_manager.cc:616] Registered 0 tablets
08:07:23.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:23.820895 8401 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
08:07:23.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:23.976249 8401 rpc_server.cc:307] RPC server started. Bound to: 127.7.161.195:37403
08:07:23.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:23.976326 8531 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.7.161.195:37403 every 8 connection(s)
08:07:23.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:23.978873 8401 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster3944160619542668700/ts-2/data/info.pb
08:07:23.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:23.985139 7815 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskcrgkbR/build/tsan/bin/kudu as pid 8401
08:07:23.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:23.985633 7815 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster3944160619542668700/ts-2/wal/instance
08:07:24.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:24.045006 8533 heartbeater.cc:344] Connected to a master server at 127.7.161.253:44339
08:07:24.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:24.045207 8534 heartbeater.cc:344] Connected to a master server at 127.7.161.254:39651
08:07:24.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:24.045526 8533 heartbeater.cc:461] Registering TS with master...
08:07:24.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:24.046000 8534 heartbeater.cc:461] Registering TS with master...
08:07:24.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:24.046123 8532 heartbeater.cc:344] Connected to a master server at 127.7.161.252:36527
08:07:24.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:24.046465 8532 heartbeater.cc:461] Registering TS with master...
08:07:24.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:24.046960 8533 heartbeater.cc:507] Master 127.7.161.253:44339 requested a full tablet report, sending...
08:07:24.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:24.047595 8534 heartbeater.cc:507] Master 127.7.161.254:39651 requested a full tablet report, sending...
08:07:24.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:24.047700 8532 heartbeater.cc:507] Master 127.7.161.252:36527 requested a full tablet report, sending...
08:07:24.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:24.050279 8016 ts_manager.cc:194] Registered new tserver with Master: f34c4dd855bb4d69a094f8670a4b9a46 (127.7.161.195:37403)
08:07:24.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:24.050839 7935 ts_manager.cc:194] Registered new tserver with Master: f34c4dd855bb4d69a094f8670a4b9a46 (127.7.161.195:37403)
08:07:24.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:24.051352 7859 ts_manager.cc:194] Registered new tserver with Master: f34c4dd855bb4d69a094f8670a4b9a46 (127.7.161.195:37403)
08:07:24.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:24.052150 8016 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.7.161.195:57777
08:07:24.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:24.060433 7815 external_mini_cluster.cc:949] 3 TS(s) registered with all masters
08:07:24.074 [DEBUG - main] (MiniKuduCluster.java:195) Response:
08:07:24.075 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
08:07:24.087 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.7.161.254"
port: 39651
}
bound_http_address {
host: "127.7.161.254"
port: 37713
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.7.161.253"
port: 44339
}
bound_http_address {
host: "127.7.161.253"
port: 33521
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.7.161.252"
port: 36527
}
bound_http_address {
host: "127.7.161.252"
port: 35917
}
}
}
08:07:24.090 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
08:07:24.093 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.7.161.193"
port: 33937
}
bound_http_address {
host: "127.7.161.193"
port: 34939
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.7.161.194"
port: 33697
}
bound_http_address {
host: "127.7.161.194"
port: 44143
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.7.161.195"
port: 37403
}
bound_http_address {
host: "127.7.161.195"
port: 34755
}
}
}
08:07:24.093 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
08:07:24.255 [DEBUG - main] (CompositeStrategy.java:59) Unable to load modern Subject methods, relying only on legacy methods
java.lang.NoSuchMethodException: javax.security.auth.Subject.current()
at java.lang.Class.getDeclaredMethod(Class.java:2130) ~[?:1.8.0_275]
at org.apache.kudu.client.internals.ModernStrategy.<init>(ModernStrategy.java:51) ~[main/:?]
at org.apache.kudu.client.internals.CompositeStrategy.<init>(CompositeStrategy.java:52) [main/:?]
at org.apache.kudu.client.internals.CompositeStrategy.<clinit>(CompositeStrategy.java:41) [main/:?]
at org.apache.kudu.client.internals.SecurityManagerCompatibility.get(SecurityManagerCompatibility.java:43) [main/:?]
at org.apache.kudu.client.SecurityContext.setupSubject(SecurityContext.java:155) [main/:?]
at org.apache.kudu.client.SecurityContext.<init>(SecurityContext.java:138) [main/:?]
at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:431) [main/:?]
at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:299) [main/:?]
at org.apache.kudu.client.AsyncKuduClient$AsyncKuduClientBuilder.build(AsyncKuduClient.java:3109) [main/:?]
at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:161) [kudu-test-utils-1.19.0-SNAPSHOT-unshaded.jar:1.19.0-SNAPSHOT]
at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50) [junit-4.13.2.jar:4.13.2]
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184) [kudu-test-utils-1.19.0-SNAPSHOT-unshaded.jar:1.19.0-SNAPSHOT]
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244) [kudu-test-utils-1.19.0-SNAPSHOT-unshaded.jar:1.19.0-SNAPSHOT]
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184) [kudu-test-utils-1.19.0-SNAPSHOT-unshaded.jar:1.19.0-SNAPSHOT]
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244) [kudu-test-utils-1.19.0-SNAPSHOT-unshaded.jar:1.19.0-SNAPSHOT]
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.run(ParentRunner.java:413) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.Suite.runChild(Suite.java:128) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.Suite.runChild(Suite.java:27) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.run(ParentRunner.java:413) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.run(JUnitCore.java:137) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.run(JUnitCore.java:115) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.main(JUnitCore.java:36) [junit-4.13.2.jar:4.13.2]
java.lang.NoSuchMethodException: javax.security.auth.Subject.current()
at java.lang.Class.getDeclaredMethod(Class.java:2130)
at org.apache.kudu.client.internals.ModernStrategy.<init>(ModernStrategy.java:51)
at org.apache.kudu.client.internals.CompositeStrategy.<init>(CompositeStrategy.java:52)
at org.apache.kudu.client.internals.CompositeStrategy.<clinit>(CompositeStrategy.java:41)
at org.apache.kudu.client.internals.SecurityManagerCompatibility.get(SecurityManagerCompatibility.java:43)
at org.apache.kudu.client.SecurityContext.setupSubject(SecurityContext.java:155)
at org.apache.kudu.client.SecurityContext.<init>(SecurityContext.java:138)
at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:431)
at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$AsyncKuduClientBuilder.build(AsyncKuduClient.java:3109)
at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:161)
at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runners.Suite.runChild(Suite.java:128)
at org.junit.runners.Suite.runChild(Suite.java:27)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runner.JUnitCore.run(JUnitCore.java:137)
at org.junit.runner.JUnitCore.run(JUnitCore.java:115)
at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77)
at org.junit.runner.JUnitCore.main(JUnitCore.java:36)
08:07:24.280 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
08:07:24.400 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskcrgkbR/test-tmp
08:07:24.400 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
08:07:24.401 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskcrgkbR/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
08:07:24.403 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683"
extra_master_flags: "--webserver_require_spnego=false"
extra_master_flags: "--authn_token_validity_seconds=10"
extra_tserver_flags: "--webserver_require_spnego=false"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
08:07:24.743 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:07:24.743 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
08:07:24.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:24.746891 8543 mini_oidc.cc:150] Starting JWKS server
08:07:24.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:24.750240 8543 webserver.cc:492] Webserver started at http://127.0.0.1:40533/ using document root <none> and password file <none>
08:07:24.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:24.750640 8543 mini_oidc.cc:186] Starting OIDC Discovery server
08:07:24.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:24.752830 8543 webserver.cc:492] Webserver started at http://127.0.0.1:34951/ using document root <none> and password file <none>
08:07:24.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
08:07:24.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/krb5kdc/principal' for realm 'KRBTEST.COM',
08:07:24.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
08:07:24.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 23 08:07:24 dist-test-slave-wxqn krb5kdc[8555](info): setting up network...
08:07:24.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
08:07:24.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 23 08:07:24 dist-test-slave-wxqn krb5kdc[8555](info): set up 2 sockets
08:07:24.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 23 08:07:24 dist-test-slave-wxqn krb5kdc[8555](info): commencing operation
08:07:24.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
08:07:25.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:25.055465 8532 heartbeater.cc:499] Master 127.7.161.252:36527 was elected leader, sending a full tablet report...
08:07:26.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:26.844650 8543 mini_kdc.cc:121] Time spent starting KDC: real 2.055s user 0.001s sys 0.019s
08:07:26.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
08:07:26.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
08:07:26.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
08:07:26.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
08:07:26.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
08:07:26.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
08:07:26.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
08:07:26.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
08:07:26.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
08:07:26.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
08:07:26.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/krb5kdc/test-user.keytab.
08:07:26.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/krb5kdc/test-user.keytab.
08:07:26.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 23 08:07:26 dist-test-slave-wxqn krb5kdc[8555](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761206846, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:07:26.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
08:07:26.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-10-23T08:07:26Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
08:07:26.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-10-23T08:07:26Z Disabled control of system clock
08:07:27.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.8.87.254@KRBTEST.COM; defaulting to no policy
08:07:27.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:07:27.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.8.87.254@KRBTEST.COM" created.
08:07:27.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:07:27.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.8.87.254 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/krb5kdc/kudu.keytab.
08:07:27.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.8.87.254 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/krb5kdc/kudu.keytab.
08:07:27.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.8.87.254@KRBTEST.COM; defaulting to no policy
08:07:27.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:07:27.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.8.87.254@KRBTEST.COM" created.
08:07:27.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:07:27.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.8.87.254 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/krb5kdc/kudu.keytab.
08:07:27.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.8.87.254 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/krb5kdc/kudu.keytab.
08:07:27.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:27.070148 8543 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskcrgkbR/build/tsan/bin/kudu
08:07:27.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskcrgkbR/build/tsan/bin/kudu
08:07:27.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-0/wal
08:07:27.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-0/data
08:07:27.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:07:27.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:07:27.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:07:27.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:07:27.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:07:27.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:07:27.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-0/logs
08:07:27.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-0/data/info.pb
08:07:27.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:07:27.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:07:27.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:07:27.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:07:27.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:07:27.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:07:27.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:07:27.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:07:27.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:07:27.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:07:27.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:07:27.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:07:27.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.87.254:43115
08:07:27.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.87.254
08:07:27.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:07:27.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/krb5kdc/kudu.keytab
08:07:27.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.8.87.254
08:07:27.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:07:27.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
08:07:27.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
08:07:27.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
08:07:27.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.87.212:46335
08:07:27.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:07:27.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:07:27.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:07:27.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:07:27.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.87.254:43115,127.8.87.253:46439,127.8.87.252:38507
08:07:27.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:07:27.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40533/jwks.json
08:07:27.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
08:07:27.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
08:07:27.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:27.390513 8573 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:07:27.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:27.391047 8573 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:07:27.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:27.391677 8573 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:07:27.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:27.425205 8573 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
08:07:27.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:27.425449 8573 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
08:07:27.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:27.425709 8573 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:07:27.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:27.425910 8573 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
08:07:27.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:27.426098 8573 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
08:07:27.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:27.426285 8573 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
08:07:27.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:27.426437 8573 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:40533/jwks.json
08:07:27.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:27.466146 8573 master_runner.cc:386] Master server non-default flags:
08:07:27.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:07:27.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.87.212:46335
08:07:27.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:07:27.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:07:27.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-0/data
08:07:27.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-0/wal
08:07:27.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
08:07:27.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:07:27.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.87.254:43115,127.8.87.253:46439,127.8.87.252:38507
08:07:27.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/krb5kdc/kudu.keytab
08:07:27.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.8.87.254
08:07:27.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:07:27.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:07:27.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:07:27.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.87.254:43115
08:07:27.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:07:27.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:07:27.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:07:27.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40533/jwks.json
08:07:27.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:07:27.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
08:07:27.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
08:07:27.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:07:27.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:07:27.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-0/data/info.pb
08:07:27.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.87.254
08:07:27.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:07:27.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:07:27.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:07:27.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:07:27.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:07:27.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:07:27.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-0/logs
08:07:27.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:07:27.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:07:27.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:07:27.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:07:27.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:07:27.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 18b1930e830609248ee2e26d6216d073ff40ac5d
08:07:27.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:07:27.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 23 Oct 2025 07:43:21 UTC on 5fd53c4cbb9d
08:07:27.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8677
08:07:27.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:07:27.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:27.467388 8573 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:07:27.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:27.468428 8573 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:07:27.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:27.479646 8580 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:07:27.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:27.734150 8249 debug-util.cc:398] Leaking SignalData structure 0x7b08000b7d40 after lost signal to thread 8120
08:07:27.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:27.735909 8249 debug-util.cc:398] Leaking SignalData structure 0x7b08000b72a0 after lost signal to thread 8252
08:07:29.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:28.888924 8579 debug-util.cc:398] Leaking SignalData structure 0x7b0800027080 after lost signal to thread 8573
08:07:29.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:29.263088 8579 kernel_stack_watchdog.cc:198] Thread 8573 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 402ms:
08:07:29.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
08:07:29.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
08:07:29.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
08:07:29.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
08:07:29.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:27.481825 8581 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:07:29.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:29.264580 8573 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.784s user 0.678s sys 0.995s
08:07:29.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:29.265151 8573 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.784s user 0.678s sys 0.995s
08:07:29.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:29.266608 8583 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:07:29.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:29.269493 8582 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection timed out after 1788 milliseconds
08:07:29.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:29.269536 8573 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
08:07:29.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:29.271122 8573 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:07:29.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:29.274262 8573 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:07:29.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:29.275746 8573 hybrid_clock.cc:648] HybridClock initialized: now 1761206849275641 us; error 102 us; skew 500 ppm
08:07:29.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:29.276964 8573 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:07:29.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 23 08:07:29 dist-test-slave-wxqn krb5kdc[8555](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761206849, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.87.254@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:07:29.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:29.284592 8573 init.cc:377] Logged in from keytab as kudu/127.8.87.254@KRBTEST.COM (short username kudu)
08:07:29.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:29.290953 8573 webserver.cc:492] Webserver started at http://127.8.87.254:33423/ using document root <none> and password file <none>
08:07:29.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:29.292213 8573 fs_manager.cc:362] Metadata directory not provided
08:07:29.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:29.292511 8573 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:07:29.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:29.293066 8573 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
08:07:29.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:29.299690 8573 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-0/data/instance:
08:07:29.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "62e54dd7499440acaa1b1c76403da552"
08:07:29.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-23 08:07:29 on dist-test-slave-wxqn"
08:07:29.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:29.301146 8573 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-0/wal/instance:
08:07:29.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "62e54dd7499440acaa1b1c76403da552"
08:07:29.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-23 08:07:29 on dist-test-slave-wxqn"
08:07:29.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:29.311383 8573 fs_manager.cc:696] Time spent creating directory manager: real 0.010s user 0.005s sys 0.005s
08:07:29.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:29.319331 8592 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:07:29.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:29.320650 8573 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.006s sys 0.000s
08:07:29.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:29.321064 8573 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-0/data,/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-0/wal
08:07:29.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "62e54dd7499440acaa1b1c76403da552"
08:07:29.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-23 08:07:29 on dist-test-slave-wxqn"
08:07:29.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:29.321503 8573 fs_report.cc:389] FS layout report
08:07:29.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:07:29.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-0/wal
08:07:29.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-0/wal
08:07:29.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-0/data/data
08:07:29.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:07:29.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:07:29.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:07:29.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:07:29.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:07:29.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:07:29.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:07:29.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:07:29.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:07:29.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:07:29.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:07:29.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:29.381851 8573 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:07:29.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:29.391736 8573 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:07:29.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:29.392275 8573 kserver.cc:163] Server-wide thread pool size limit: 3276
08:07:29.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:29.505154 8573 rpc_server.cc:307] RPC server started. Bound to: 127.8.87.254:43115
08:07:29.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:29.505173 8643 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.8.87.254:43115 every 8 connection(s)
08:07:29.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:29.508841 8573 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-0/data/info.pb
08:07:29.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:29.511452 8543 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskcrgkbR/build/tsan/bin/kudu as pid 8573
08:07:29.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:29.511976 8543 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-0/wal/instance
08:07:29.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:29.526937 8644 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
08:07:29.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.8.87.253@KRBTEST.COM; defaulting to no policy
08:07:29.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:29.541013 8644 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.87.254" port: 43115 } has no permanent_uuid. Determining permanent_uuid...
08:07:29.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:07:29.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.8.87.253@KRBTEST.COM" created.
08:07:29.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 23 08:07:29 dist-test-slave-wxqn krb5kdc[8555](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761206849, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.87.254@KRBTEST.COM for kudu/127.8.87.254@KRBTEST.COM
08:07:29.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:07:29.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.8.87.253 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/krb5kdc/kudu.keytab.
08:07:29.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.8.87.253 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/krb5kdc/kudu.keytab.
08:07:29.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:29.584219 8644 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.87.253" port: 46439 } has no permanent_uuid. Determining permanent_uuid...
08:07:29.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:29.588440 8596 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.87.253:46439: connect: Connection refused (error 111)
08:07:29.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.8.87.253@KRBTEST.COM; defaulting to no policy
08:07:29.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:07:29.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.8.87.253@KRBTEST.COM" created.
08:07:29.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:29.592960 8644 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.8.87.253:46439: Network error: Client connection negotiation failed: client connection to 127.8.87.253:46439: connect: Connection refused (error 111)
08:07:29.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:07:29.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.8.87.253 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/krb5kdc/kudu.keytab.
08:07:29.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.8.87.253 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/krb5kdc/kudu.keytab.
08:07:29.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:29.615046 8543 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskcrgkbR/build/tsan/bin/kudu
08:07:29.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskcrgkbR/build/tsan/bin/kudu
08:07:29.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-1/wal
08:07:29.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-1/data
08:07:29.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:07:29.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:07:29.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:07:29.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:07:29.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:07:29.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:07:29.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-1/logs
08:07:29.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-1/data/info.pb
08:07:29.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:07:29.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:07:29.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:07:29.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:07:29.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:07:29.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:07:29.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:07:29.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:07:29.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:07:29.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:07:29.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:07:29.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:07:29.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.87.253:46439
08:07:29.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.87.253
08:07:29.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:07:29.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/krb5kdc/kudu.keytab
08:07:29.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.8.87.253
08:07:29.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:07:29.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
08:07:29.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
08:07:29.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
08:07:29.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.87.212:46335
08:07:29.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:07:29.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:07:29.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:07:29.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:07:29.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.87.254:43115,127.8.87.253:46439,127.8.87.252:38507
08:07:29.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:07:29.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40533/jwks.json
08:07:29.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
08:07:29.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
08:07:29.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:29.642527 8644 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.87.253" port: 46439 } attempt: 1
08:07:29.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:29.648110 8644 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.8.87.253:46439: Network error: Client connection negotiation failed: client connection to 127.8.87.253:46439: connect: Connection refused (error 111)
08:07:29.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:29.716574 8644 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.87.253" port: 46439 } attempt: 2
08:07:29.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:29.721455 8644 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.8.87.253:46439: Network error: Client connection negotiation failed: client connection to 127.8.87.253:46439: connect: Connection refused (error 111)
08:07:29.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:29.812938 8644 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.87.253" port: 46439 } attempt: 3
08:07:29.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:29.818181 8644 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.8.87.253:46439: Network error: Client connection negotiation failed: client connection to 127.8.87.253:46439: connect: Connection refused (error 111)
08:07:29.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:29.961627 8644 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.87.253" port: 46439 } attempt: 4
08:07:29.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:29.966315 8644 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.8.87.253:46439: Network error: Client connection negotiation failed: client connection to 127.8.87.253:46439: connect: Connection refused (error 111)
08:07:29.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:29.969247 8654 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:07:29.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:29.969915 8654 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:07:29.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:29.970679 8654 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:07:30.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:30.008401 8654 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
08:07:30.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:30.008692 8654 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
08:07:30.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:30.009104 8654 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:07:30.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:30.009435 8654 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
08:07:30.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:30.009732 8654 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
08:07:30.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:30.010157 8654 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
08:07:30.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:30.010392 8654 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:40533/jwks.json
08:07:30.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:30.052034 8654 master_runner.cc:386] Master server non-default flags:
08:07:30.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:07:30.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.87.212:46335
08:07:30.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:07:30.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:07:30.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-1/data
08:07:30.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-1/wal
08:07:30.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
08:07:30.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:07:30.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.87.254:43115,127.8.87.253:46439,127.8.87.252:38507
08:07:30.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/krb5kdc/kudu.keytab
08:07:30.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.8.87.253
08:07:30.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:07:30.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:07:30.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:07:30.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.87.253:46439
08:07:30.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:07:30.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:07:30.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:07:30.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40533/jwks.json
08:07:30.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:07:30.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
08:07:30.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
08:07:30.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:07:30.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:07:30.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-1/data/info.pb
08:07:30.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.87.253
08:07:30.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:07:30.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:07:30.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:07:30.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:07:30.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:07:30.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:07:30.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-1/logs
08:07:30.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:07:30.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:07:30.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:07:30.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:07:30.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:07:30.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 18b1930e830609248ee2e26d6216d073ff40ac5d
08:07:30.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:07:30.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 23 Oct 2025 07:43:21 UTC on 5fd53c4cbb9d
08:07:30.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8677
08:07:30.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:07:30.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:30.053668 8654 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:07:30.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:30.054945 8654 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:07:30.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:30.068521 8660 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:07:30.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:30.265784 8644 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.87.253" port: 46439 } attempt: 5
08:07:30.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:30.273468 8644 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.8.87.253:46439: Network error: Client connection negotiation failed: client connection to 127.8.87.253:46439: connect: Connection refused (error 111)
08:07:30.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:30.821076 8644 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.87.253" port: 46439 } attempt: 6
08:07:30.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:30.855430 8644 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.8.87.253:46439: Network error: Client connection negotiation failed: client connection to 127.8.87.253:46439: connect: Connection refused (error 111)
08:07:31.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:31.915911 8644 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.87.253" port: 46439 } attempt: 7
08:07:31.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:31.942988 8644 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.8.87.253:46439: Network error: Client connection negotiation failed: client connection to 127.8.87.253:46439: connect: Connection refused (error 111)
08:07:31.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:31.467873 8659 debug-util.cc:398] Leaking SignalData structure 0x7b0800027080 after lost signal to thread 8654
08:07:31.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:31.971191 8659 kernel_stack_watchdog.cc:198] Thread 8654 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 395ms:
08:07:31.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
08:07:31.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
08:07:31.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
08:07:31.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
08:07:31.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:31.972028 8654 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.902s user 0.570s sys 1.215s
08:07:31.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:30.071375 8661 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:07:31.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:31.972720 8654 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.903s user 0.571s sys 1.215s
08:07:31.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:31.974256 8663 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:07:31.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:31.976917 8662 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection timed out after 1906 milliseconds
08:07:31.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:31.976945 8654 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
08:07:31.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:31.978381 8654 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:07:31.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:31.982017 8654 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:07:31.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:31.983487 8654 hybrid_clock.cc:648] HybridClock initialized: now 1761206851983373 us; error 123 us; skew 500 ppm
08:07:31.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:31.984562 8654 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:07:31.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 23 08:07:31 dist-test-slave-wxqn krb5kdc[8555](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761206851, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.87.253@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:07:31.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:31.991571 8654 init.cc:377] Logged in from keytab as kudu/127.8.87.253@KRBTEST.COM (short username kudu)
08:07:31.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:31.997395 8654 webserver.cc:492] Webserver started at http://127.8.87.253:44589/ using document root <none> and password file <none>
08:07:31.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:31.998514 8654 fs_manager.cc:362] Metadata directory not provided
08:07:31.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:31.998746 8654 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:07:31.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:31.999212 8654 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
08:07:32.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:32.004477 8654 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-1/data/instance:
08:07:32.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4283b8cd94454e13bb77279c6afdb8f2"
08:07:32.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-23 08:07:31 on dist-test-slave-wxqn"
08:07:32.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:32.006114 8654 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-1/wal/instance:
08:07:32.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4283b8cd94454e13bb77279c6afdb8f2"
08:07:32.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-23 08:07:31 on dist-test-slave-wxqn"
08:07:32.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:32.013665 8654 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.006s sys 0.002s
08:07:32.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:32.020365 8680 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:07:32.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:32.021517 8654 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
08:07:32.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:32.021818 8654 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-1/data,/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-1/wal
08:07:32.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4283b8cd94454e13bb77279c6afdb8f2"
08:07:32.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-23 08:07:31 on dist-test-slave-wxqn"
08:07:32.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:32.022107 8654 fs_report.cc:389] FS layout report
08:07:32.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:07:32.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-1/wal
08:07:32.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-1/wal
08:07:32.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-1/data/data
08:07:32.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:07:32.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:07:32.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:07:32.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:07:32.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:07:32.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:07:32.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:07:32.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:07:32.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:07:32.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:07:32.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:07:32.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:32.073549 8654 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:07:32.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:32.079043 8654 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:07:32.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:32.079442 8654 kserver.cc:163] Server-wide thread pool size limit: 3276
08:07:32.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:32.173921 8654 rpc_server.cc:307] RPC server started. Bound to: 127.8.87.253:46439
08:07:32.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:32.173995 8731 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.8.87.253:46439 every 8 connection(s)
08:07:32.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:32.176435 8654 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-1/data/info.pb
08:07:32.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:32.179437 8543 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskcrgkbR/build/tsan/bin/kudu as pid 8654
08:07:32.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:32.179972 8543 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-1/wal/instance
08:07:32.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:32.182998 8732 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
08:07:32.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.8.87.252@KRBTEST.COM; defaulting to no policy
08:07:32.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:32.194741 8732 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.87.254" port: 43115 } has no permanent_uuid. Determining permanent_uuid...
08:07:32.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:07:32.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.8.87.252@KRBTEST.COM" created.
08:07:32.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 23 08:07:32 dist-test-slave-wxqn krb5kdc[8555](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761206851, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.87.253@KRBTEST.COM for kudu/127.8.87.254@KRBTEST.COM
08:07:32.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:07:32.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.8.87.252 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/krb5kdc/kudu.keytab.
08:07:32.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.8.87.252 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/krb5kdc/kudu.keytab.
08:07:32.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:32.234210 8732 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.87.253" port: 46439 } has no permanent_uuid. Determining permanent_uuid...
08:07:32.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.8.87.252@KRBTEST.COM; defaulting to no policy
08:07:32.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:07:32.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.8.87.252@KRBTEST.COM" created.
08:07:32.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 23 08:07:32 dist-test-slave-wxqn krb5kdc[8555](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761206851, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.87.253@KRBTEST.COM for kudu/127.8.87.253@KRBTEST.COM
08:07:32.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:32.262017 8732 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.87.252" port: 38507 } has no permanent_uuid. Determining permanent_uuid...
08:07:32.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:07:32.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.8.87.252 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/krb5kdc/kudu.keytab.
08:07:32.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.8.87.252 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/krb5kdc/kudu.keytab.
08:07:32.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:32.264546 8683 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.87.252:38507: connect: Connection refused (error 111)
08:07:32.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:32.266983 8543 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskcrgkbR/build/tsan/bin/kudu
08:07:32.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskcrgkbR/build/tsan/bin/kudu
08:07:32.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-2/wal
08:07:32.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-2/data
08:07:32.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:07:32.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:07:32.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:07:32.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:07:32.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:07:32.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:07:32.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-2/logs
08:07:32.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-2/data/info.pb
08:07:32.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:07:32.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:07:32.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:07:32.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:07:32.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:07:32.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:07:32.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:07:32.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:07:32.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:07:32.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:07:32.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:07:32.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:07:32.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.87.252:38507
08:07:32.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.87.252
08:07:32.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:07:32.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/krb5kdc/kudu.keytab
08:07:32.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.8.87.252
08:07:32.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:07:32.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
08:07:32.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
08:07:32.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
08:07:32.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.87.212:46335
08:07:32.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:07:32.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:07:32.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:07:32.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:07:32.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.87.254:43115,127.8.87.253:46439,127.8.87.252:38507
08:07:32.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:07:32.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40533/jwks.json
08:07:32.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
08:07:32.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
08:07:32.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:32.270406 8732 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.8.87.252:38507: Network error: Client connection negotiation failed: client connection to 127.8.87.252:38507: connect: Connection refused (error 111)
08:07:32.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:32.319962 8732 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.87.252" port: 38507 } attempt: 1
08:07:32.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:32.324378 8732 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.8.87.252:38507: Network error: Client connection negotiation failed: client connection to 127.8.87.252:38507: connect: Connection refused (error 111)
08:07:32.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:32.392750 8732 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.87.252" port: 38507 } attempt: 2
08:07:32.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:32.396595 8732 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.8.87.252:38507: Network error: Client connection negotiation failed: client connection to 127.8.87.252:38507: connect: Connection refused (error 111)
08:07:32.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:32.488122 8732 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.87.252" port: 38507 } attempt: 3
08:07:32.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:32.492262 8732 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.8.87.252:38507: Network error: Client connection negotiation failed: client connection to 127.8.87.252:38507: connect: Connection refused (error 111)
08:07:32.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:32.594790 8742 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:07:32.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:32.595340 8742 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:07:32.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:32.595966 8742 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:07:32.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:32.628166 8742 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
08:07:32.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:32.628512 8742 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
08:07:32.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:32.628957 8742 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:07:32.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:32.629278 8742 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
08:07:32.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:32.629594 8742 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
08:07:32.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:32.629899 8742 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
08:07:32.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:32.630127 8742 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:40533/jwks.json
08:07:32.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:32.635663 8732 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.87.252" port: 38507 } attempt: 4
08:07:32.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:32.640354 8732 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.8.87.252:38507: Network error: Client connection negotiation failed: client connection to 127.8.87.252:38507: connect: Connection refused (error 111)
08:07:32.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:32.695410 8742 master_runner.cc:386] Master server non-default flags:
08:07:32.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:07:32.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.87.212:46335
08:07:32.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:07:32.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:07:32.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-2/data
08:07:32.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-2/wal
08:07:32.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
08:07:32.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:07:32.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.87.254:43115,127.8.87.253:46439,127.8.87.252:38507
08:07:32.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/krb5kdc/kudu.keytab
08:07:32.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.8.87.252
08:07:32.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:07:32.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:07:32.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:07:32.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.87.252:38507
08:07:32.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:07:32.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:07:32.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:07:32.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40533/jwks.json
08:07:32.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:07:32.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
08:07:32.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
08:07:32.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:07:32.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:07:32.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-2/data/info.pb
08:07:32.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.87.252
08:07:32.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:07:32.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:07:32.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:07:32.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:07:32.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:07:32.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:07:32.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-2/logs
08:07:32.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:07:32.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:07:32.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:07:32.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:07:32.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:07:32.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 18b1930e830609248ee2e26d6216d073ff40ac5d
08:07:32.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:07:32.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 23 Oct 2025 07:43:21 UTC on 5fd53c4cbb9d
08:07:32.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8677
08:07:32.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:07:32.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:32.697245 8742 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:07:32.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:32.698717 8742 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:07:32.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:32.714026 8749 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:07:32.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:32.939837 8732 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.87.252" port: 38507 } attempt: 5
08:07:32.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:32.953871 8732 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.8.87.252:38507: Network error: Client connection negotiation failed: client connection to 127.8.87.252:38507: connect: Connection refused (error 111)
08:07:33.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:33.502329 8732 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.87.252" port: 38507 } attempt: 6
08:07:33.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:33.525691 8732 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.8.87.252:38507: Network error: Client connection negotiation failed: client connection to 127.8.87.252:38507: connect: Connection refused (error 111)
08:07:33.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:33.723806 8047 debug-util.cc:398] Leaking SignalData structure 0x7b080009b960 after lost signal to thread 7977
08:07:33.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:33.724800 8047 debug-util.cc:398] Leaking SignalData structure 0x7b08000b33c0 after lost signal to thread 8050
08:07:34.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:34.033576 8644 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.87.253" port: 46439 } attempt: 8
08:07:34.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 23 08:07:34 dist-test-slave-wxqn krb5kdc[8555](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761206849, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.87.254@KRBTEST.COM for kudu/127.8.87.253@KRBTEST.COM
08:07:34.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:34.078264 8644 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.87.252" port: 38507 } has no permanent_uuid. Determining permanent_uuid...
08:07:34.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:34.096122 8644 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.8.87.252:38507: Network error: Client connection negotiation failed: client connection to 127.8.87.252:38507: connect: Connection refused (error 111)
08:07:34.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:34.163815 8644 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.87.252" port: 38507 } attempt: 1
08:07:34.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:34.184450 8644 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.8.87.252:38507: Network error: Client connection negotiation failed: client connection to 127.8.87.252:38507: connect: Connection refused (error 111)
08:07:34.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:34.118623 8748 debug-util.cc:398] Leaking SignalData structure 0x7b0800027080 after lost signal to thread 8742
08:07:34.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:34.237844 8742 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 1.519s user 0.654s sys 0.861s
08:07:34.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:34.238149 8644 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.87.252" port: 38507 } attempt: 2
08:07:34.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:34.238523 8742 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 1.519s user 0.654s sys 0.862s
08:07:34.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:32.721205 8750 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:07:34.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:34.244022 8644 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.8.87.252:38507: Network error: Client connection negotiation failed: client connection to 127.8.87.252:38507: connect: Connection refused (error 111)
08:07:34.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:34.247772 8742 server_base.cc:1047] running on GCE node
08:07:34.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:34.248734 8761 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:07:34.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:34.249995 8742 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:07:34.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:34.253474 8742 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:07:34.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:34.254877 8742 hybrid_clock.cc:648] HybridClock initialized: now 1761206854254828 us; error 54 us; skew 500 ppm
08:07:34.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:34.255731 8742 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:07:34.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 23 08:07:34 dist-test-slave-wxqn krb5kdc[8555](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761206854, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.87.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:07:34.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:34.263720 8742 init.cc:377] Logged in from keytab as kudu/127.8.87.252@KRBTEST.COM (short username kudu)
08:07:34.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:34.270252 8742 webserver.cc:492] Webserver started at http://127.8.87.252:40629/ using document root <none> and password file <none>
08:07:34.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:34.271332 8742 fs_manager.cc:362] Metadata directory not provided
08:07:34.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:34.271577 8742 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:07:34.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:34.272121 8742 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
08:07:34.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:34.276747 8742 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-2/data/instance:
08:07:34.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b0ae399fb47a454996698251cea1c12d"
08:07:34.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-23 08:07:34 on dist-test-slave-wxqn"
08:07:34.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:34.277989 8742 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-2/wal/instance:
08:07:34.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b0ae399fb47a454996698251cea1c12d"
08:07:34.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-23 08:07:34 on dist-test-slave-wxqn"
08:07:34.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:34.286779 8742 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.002s sys 0.006s
08:07:34.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:34.293007 8767 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:07:34.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:34.295131 8742 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.001s sys 0.003s
08:07:34.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:34.295449 8742 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-2/data,/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-2/wal
08:07:34.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b0ae399fb47a454996698251cea1c12d"
08:07:34.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-23 08:07:34 on dist-test-slave-wxqn"
08:07:34.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:34.295771 8742 fs_report.cc:389] FS layout report
08:07:34.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:07:34.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-2/wal
08:07:34.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-2/wal
08:07:34.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-2/data/data
08:07:34.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:07:34.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:07:34.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:07:34.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:07:34.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:07:34.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:07:34.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:07:34.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:07:34.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:07:34.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:07:34.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:07:34.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:34.320542 8644 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.87.252" port: 38507 } attempt: 3
08:07:34.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:34.325748 8644 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.8.87.252:38507: Network error: Client connection negotiation failed: client connection to 127.8.87.252:38507: connect: Connection refused (error 111)
08:07:34.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:34.350375 8742 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:07:34.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:34.356828 8742 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:07:34.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:34.357411 8742 kserver.cc:163] Server-wide thread pool size limit: 3276
08:07:34.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:34.471151 8742 rpc_server.cc:307] RPC server started. Bound to: 127.8.87.252:38507
08:07:34.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:34.471166 8818 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.8.87.252:38507 every 8 connection(s)
08:07:34.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:34.475504 8742 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-2/data/info.pb
08:07:34.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:34.476298 8543 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskcrgkbR/build/tsan/bin/kudu as pid 8742
08:07:34.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:34.476784 8543 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-2/wal/instance
08:07:34.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:34.481784 8644 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.87.252" port: 38507 } attempt: 4
08:07:34.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:34.486042 8819 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
08:07:34.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:34.531436 8819 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.87.254" port: 43115 } has no permanent_uuid. Determining permanent_uuid...
08:07:34.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 23 08:07:34 dist-test-slave-wxqn krb5kdc[8555](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761206849, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.87.254@KRBTEST.COM for kudu/127.8.87.252@KRBTEST.COM
08:07:34.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 23 08:07:34 dist-test-slave-wxqn krb5kdc[8555](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761206846, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.8.87.254@KRBTEST.COM
08:07:34.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 23 08:07:34 dist-test-slave-wxqn krb5kdc[8555](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761206854, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.87.252@KRBTEST.COM for kudu/127.8.87.254@KRBTEST.COM
08:07:34.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:34.579463 8644 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 62e54dd7499440acaa1b1c76403da552: Bootstrap starting.
08:07:34.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:34.586242 8732 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.87.252" port: 38507 } attempt: 7
08:07:34.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 23 08:07:34 dist-test-slave-wxqn krb5kdc[8555](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761206846, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.8.87.253@KRBTEST.COM
08:07:34.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:34.596175 8644 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 62e54dd7499440acaa1b1c76403da552: Neither blocks nor log segments found. Creating new log.
08:07:34.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:34.598306 8819 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.87.253" port: 46439 } has no permanent_uuid. Determining permanent_uuid...
08:07:34.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:34.600936 8644 log.cc:826] T 00000000000000000000000000000000 P 62e54dd7499440acaa1b1c76403da552: Log is configured to *not* fsync() on all Append() calls
08:07:34.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:34.615803 8644 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 62e54dd7499440acaa1b1c76403da552: No bootstrap required, opened a new log
08:07:34.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 23 08:07:34 dist-test-slave-wxqn krb5kdc[8555](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761206854, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.87.252@KRBTEST.COM for kudu/127.8.87.253@KRBTEST.COM
08:07:34.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 23 08:07:34 dist-test-slave-wxqn krb5kdc[8555](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761206851, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.87.253@KRBTEST.COM for kudu/127.8.87.252@KRBTEST.COM
08:07:34.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:34.645810 8644 raft_consensus.cc:359] T 00000000000000000000000000000000 P 62e54dd7499440acaa1b1c76403da552 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "62e54dd7499440acaa1b1c76403da552" member_type: VOTER last_known_addr { host: "127.8.87.254" port: 43115 } } peers { permanent_uuid: "4283b8cd94454e13bb77279c6afdb8f2" member_type: VOTER last_known_addr { host: "127.8.87.253" port: 46439 } } peers { permanent_uuid: "b0ae399fb47a454996698251cea1c12d" member_type: VOTER last_known_addr { host: "127.8.87.252" port: 38507 } }
08:07:34.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:34.647030 8644 raft_consensus.cc:385] T 00000000000000000000000000000000 P 62e54dd7499440acaa1b1c76403da552 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:07:34.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:34.647778 8644 raft_consensus.cc:740] T 00000000000000000000000000000000 P 62e54dd7499440acaa1b1c76403da552 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 62e54dd7499440acaa1b1c76403da552, State: Initialized, Role: FOLLOWER
08:07:34.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:34.648911 8644 consensus_queue.cc:260] T 00000000000000000000000000000000 P 62e54dd7499440acaa1b1c76403da552 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "62e54dd7499440acaa1b1c76403da552" member_type: VOTER last_known_addr { host: "127.8.87.254" port: 43115 } } peers { permanent_uuid: "4283b8cd94454e13bb77279c6afdb8f2" member_type: VOTER last_known_addr { host: "127.8.87.253" port: 46439 } } peers { permanent_uuid: "b0ae399fb47a454996698251cea1c12d" member_type: VOTER last_known_addr { host: "127.8.87.252" port: 38507 } }
08:07:34.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 23 08:07:34 dist-test-slave-wxqn krb5kdc[8555](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761206846, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.8.87.252@KRBTEST.COM
08:07:34.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:34.658306 8819 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.87.252" port: 38507 } has no permanent_uuid. Determining permanent_uuid...
08:07:34.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:34.675041 8732 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4283b8cd94454e13bb77279c6afdb8f2: Bootstrap starting.
08:07:34.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 23 08:07:34 dist-test-slave-wxqn krb5kdc[8555](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761206854, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.87.252@KRBTEST.COM for kudu/127.8.87.252@KRBTEST.COM
08:07:34.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:34.698341 8830 sys_catalog.cc:455] T 00000000000000000000000000000000 P 62e54dd7499440acaa1b1c76403da552 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "62e54dd7499440acaa1b1c76403da552" member_type: VOTER last_known_addr { host: "127.8.87.254" port: 43115 } } peers { permanent_uuid: "4283b8cd94454e13bb77279c6afdb8f2" member_type: VOTER last_known_addr { host: "127.8.87.253" port: 46439 } } peers { permanent_uuid: "b0ae399fb47a454996698251cea1c12d" member_type: VOTER last_known_addr { host: "127.8.87.252" port: 38507 } } }
08:07:34.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:34.699527 8830 sys_catalog.cc:458] T 00000000000000000000000000000000 P 62e54dd7499440acaa1b1c76403da552 [sys.catalog]: This master's current role is: FOLLOWER
08:07:34.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:34.704234 8732 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 4283b8cd94454e13bb77279c6afdb8f2: Neither blocks nor log segments found. Creating new log.
08:07:34.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:34.705422 8644 sys_catalog.cc:565] T 00000000000000000000000000000000 P 62e54dd7499440acaa1b1c76403da552 [sys.catalog]: configured and running, proceeding with master startup.
08:07:34.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:34.706903 8732 log.cc:826] T 00000000000000000000000000000000 P 4283b8cd94454e13bb77279c6afdb8f2: Log is configured to *not* fsync() on all Append() calls
08:07:34.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:34.740329 8819 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d: Bootstrap starting.
08:07:34.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:34.751967 8819 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d: Neither blocks nor log segments found. Creating new log.
08:07:34.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:34.754532 8819 log.cc:826] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d: Log is configured to *not* fsync() on all Append() calls
08:07:34.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:34.762210 8732 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4283b8cd94454e13bb77279c6afdb8f2: No bootstrap required, opened a new log
08:07:34.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:34.780284 8819 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d: No bootstrap required, opened a new log
08:07:34.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:34.797451 8732 raft_consensus.cc:359] T 00000000000000000000000000000000 P 4283b8cd94454e13bb77279c6afdb8f2 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "62e54dd7499440acaa1b1c76403da552" member_type: VOTER last_known_addr { host: "127.8.87.254" port: 43115 } } peers { permanent_uuid: "4283b8cd94454e13bb77279c6afdb8f2" member_type: VOTER last_known_addr { host: "127.8.87.253" port: 46439 } } peers { permanent_uuid: "b0ae399fb47a454996698251cea1c12d" member_type: VOTER last_known_addr { host: "127.8.87.252" port: 38507 } }
08:07:34.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:34.798718 8732 raft_consensus.cc:385] T 00000000000000000000000000000000 P 4283b8cd94454e13bb77279c6afdb8f2 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:07:34.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:34.799314 8732 raft_consensus.cc:740] T 00000000000000000000000000000000 P 4283b8cd94454e13bb77279c6afdb8f2 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4283b8cd94454e13bb77279c6afdb8f2, State: Initialized, Role: FOLLOWER
08:07:34.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:34.800498 8732 consensus_queue.cc:260] T 00000000000000000000000000000000 P 4283b8cd94454e13bb77279c6afdb8f2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "62e54dd7499440acaa1b1c76403da552" member_type: VOTER last_known_addr { host: "127.8.87.254" port: 43115 } } peers { permanent_uuid: "4283b8cd94454e13bb77279c6afdb8f2" member_type: VOTER last_known_addr { host: "127.8.87.253" port: 46439 } } peers { permanent_uuid: "b0ae399fb47a454996698251cea1c12d" member_type: VOTER last_known_addr { host: "127.8.87.252" port: 38507 } }
08:07:34.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:34.811653 8819 raft_consensus.cc:359] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "62e54dd7499440acaa1b1c76403da552" member_type: VOTER last_known_addr { host: "127.8.87.254" port: 43115 } } peers { permanent_uuid: "4283b8cd94454e13bb77279c6afdb8f2" member_type: VOTER last_known_addr { host: "127.8.87.253" port: 46439 } } peers { permanent_uuid: "b0ae399fb47a454996698251cea1c12d" member_type: VOTER last_known_addr { host: "127.8.87.252" port: 38507 } }
08:07:34.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:34.812712 8819 raft_consensus.cc:385] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:07:34.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:34.813051 8819 raft_consensus.cc:740] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: b0ae399fb47a454996698251cea1c12d, State: Initialized, Role: FOLLOWER
08:07:34.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:34.814090 8819 consensus_queue.cc:260] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "62e54dd7499440acaa1b1c76403da552" member_type: VOTER last_known_addr { host: "127.8.87.254" port: 43115 } } peers { permanent_uuid: "4283b8cd94454e13bb77279c6afdb8f2" member_type: VOTER last_known_addr { host: "127.8.87.253" port: 46439 } } peers { permanent_uuid: "b0ae399fb47a454996698251cea1c12d" member_type: VOTER last_known_addr { host: "127.8.87.252" port: 38507 } }
08:07:34.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:34.818133 8732 sys_catalog.cc:565] T 00000000000000000000000000000000 P 4283b8cd94454e13bb77279c6afdb8f2 [sys.catalog]: configured and running, proceeding with master startup.
08:07:34.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:34.822606 8843 sys_catalog.cc:455] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "62e54dd7499440acaa1b1c76403da552" member_type: VOTER last_known_addr { host: "127.8.87.254" port: 43115 } } peers { permanent_uuid: "4283b8cd94454e13bb77279c6afdb8f2" member_type: VOTER last_known_addr { host: "127.8.87.253" port: 46439 } } peers { permanent_uuid: "b0ae399fb47a454996698251cea1c12d" member_type: VOTER last_known_addr { host: "127.8.87.252" port: 38507 } } }
08:07:34.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:34.819264 8842 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4283b8cd94454e13bb77279c6afdb8f2 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "62e54dd7499440acaa1b1c76403da552" member_type: VOTER last_known_addr { host: "127.8.87.254" port: 43115 } } peers { permanent_uuid: "4283b8cd94454e13bb77279c6afdb8f2" member_type: VOTER last_known_addr { host: "127.8.87.253" port: 46439 } } peers { permanent_uuid: "b0ae399fb47a454996698251cea1c12d" member_type: VOTER last_known_addr { host: "127.8.87.252" port: 38507 } } }
08:07:34.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:34.823407 8843 sys_catalog.cc:458] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d [sys.catalog]: This master's current role is: FOLLOWER
08:07:34.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:34.823695 8842 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4283b8cd94454e13bb77279c6afdb8f2 [sys.catalog]: This master's current role is: FOLLOWER
08:07:34.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:34.826706 8819 sys_catalog.cc:565] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d [sys.catalog]: configured and running, proceeding with master startup.
08:07:34.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:34.838511 8686 tablet.cc:2378] T 00000000000000000000000000000000 P 4283b8cd94454e13bb77279c6afdb8f2: Can't schedule compaction. Clean time has not been advanced past its initial value.
08:07:34.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:34.858385 8851 catalog_manager.cc:805] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
08:07:34.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:34.864486 8773 tablet.cc:2378] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d: Can't schedule compaction. Clean time has not been advanced past its initial value.
08:07:34.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:34.909994 8862 catalog_manager.cc:805] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
08:07:34.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:34.911258 8598 tablet.cc:2378] T 00000000000000000000000000000000 P 62e54dd7499440acaa1b1c76403da552: Can't schedule compaction. Clean time has not been advanced past its initial value.
08:07:34.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:34.867497 8573 master_runner.cc:186] Error getting master registration for 127.8.87.253:46439: OK, instance_id { permanent_uuid: "4283b8cd94454e13bb77279c6afdb8f2" instance_seqno: 1761206852149269 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
08:07:34.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:34.919934 8573 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
08:07:34.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:34.926592 8842 raft_consensus.cc:493] T 00000000000000000000000000000000 P 4283b8cd94454e13bb77279c6afdb8f2 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:07:34.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:34.927156 8842 raft_consensus.cc:515] T 00000000000000000000000000000000 P 4283b8cd94454e13bb77279c6afdb8f2 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "62e54dd7499440acaa1b1c76403da552" member_type: VOTER last_known_addr { host: "127.8.87.254" port: 43115 } } peers { permanent_uuid: "4283b8cd94454e13bb77279c6afdb8f2" member_type: VOTER last_known_addr { host: "127.8.87.253" port: 46439 } } peers { permanent_uuid: "b0ae399fb47a454996698251cea1c12d" member_type: VOTER last_known_addr { host: "127.8.87.252" port: 38507 } }
08:07:34.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:34.933902 8842 leader_election.cc:290] T 00000000000000000000000000000000 P 4283b8cd94454e13bb77279c6afdb8f2 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 62e54dd7499440acaa1b1c76403da552 (127.8.87.254:43115), b0ae399fb47a454996698251cea1c12d (127.8.87.252:38507)
08:07:34.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:34.936034 8794 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "4283b8cd94454e13bb77279c6afdb8f2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "b0ae399fb47a454996698251cea1c12d" is_pre_election: true
08:07:34.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:34.937500 8794 raft_consensus.cc:2468] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 4283b8cd94454e13bb77279c6afdb8f2 in term 0.
08:07:34.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:34.939893 8619 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "4283b8cd94454e13bb77279c6afdb8f2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "62e54dd7499440acaa1b1c76403da552" is_pre_election: true
08:07:34.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:34.948252 8619 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 62e54dd7499440acaa1b1c76403da552 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 4283b8cd94454e13bb77279c6afdb8f2 in term 0.
08:07:34.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:34.949837 8682 leader_election.cc:304] T 00000000000000000000000000000000 P 4283b8cd94454e13bb77279c6afdb8f2 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4283b8cd94454e13bb77279c6afdb8f2, 62e54dd7499440acaa1b1c76403da552; no voters:
08:07:34.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:34.956256 8842 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 4283b8cd94454e13bb77279c6afdb8f2 [term 0 FOLLOWER]: Leader pre-election won for term 1
08:07:34.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:34.956736 8842 raft_consensus.cc:493] T 00000000000000000000000000000000 P 4283b8cd94454e13bb77279c6afdb8f2 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
08:07:34.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:34.957129 8842 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 4283b8cd94454e13bb77279c6afdb8f2 [term 0 FOLLOWER]: Advancing to term 1
08:07:34.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:34.930114 8654 master_runner.cc:186] Error getting master registration for 127.8.87.252:38507: OK, instance_id { permanent_uuid: "b0ae399fb47a454996698251cea1c12d" instance_seqno: 1761206854441192 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
08:07:34.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:34.969985 8654 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
08:07:34.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:34.970798 8842 raft_consensus.cc:515] T 00000000000000000000000000000000 P 4283b8cd94454e13bb77279c6afdb8f2 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "62e54dd7499440acaa1b1c76403da552" member_type: VOTER last_known_addr { host: "127.8.87.254" port: 43115 } } peers { permanent_uuid: "4283b8cd94454e13bb77279c6afdb8f2" member_type: VOTER last_known_addr { host: "127.8.87.253" port: 46439 } } peers { permanent_uuid: "b0ae399fb47a454996698251cea1c12d" member_type: VOTER last_known_addr { host: "127.8.87.252" port: 38507 } }
08:07:34.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:34.973654 8619 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "4283b8cd94454e13bb77279c6afdb8f2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "62e54dd7499440acaa1b1c76403da552"
08:07:34.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:34.974200 8619 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 62e54dd7499440acaa1b1c76403da552 [term 0 FOLLOWER]: Advancing to term 1
08:07:34.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:34.982388 8619 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 62e54dd7499440acaa1b1c76403da552 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 4283b8cd94454e13bb77279c6afdb8f2 in term 1.
08:07:34.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:34.983806 8682 leader_election.cc:304] T 00000000000000000000000000000000 P 4283b8cd94454e13bb77279c6afdb8f2 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4283b8cd94454e13bb77279c6afdb8f2, 62e54dd7499440acaa1b1c76403da552; no voters:
08:07:34.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:34.994105 8794 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "4283b8cd94454e13bb77279c6afdb8f2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "b0ae399fb47a454996698251cea1c12d"
08:07:34.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:34.994870 8794 raft_consensus.cc:3060] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d [term 0 FOLLOWER]: Advancing to term 1
08:07:35.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:35.003810 8842 leader_election.cc:290] T 00000000000000000000000000000000 P 4283b8cd94454e13bb77279c6afdb8f2 [CANDIDATE]: Term 1 election: Requested vote from peers 62e54dd7499440acaa1b1c76403da552 (127.8.87.254:43115), b0ae399fb47a454996698251cea1c12d (127.8.87.252:38507)
08:07:35.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:35.007385 8794 raft_consensus.cc:2468] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 4283b8cd94454e13bb77279c6afdb8f2 in term 1.
08:07:35.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:35.005785 8842 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 4283b8cd94454e13bb77279c6afdb8f2 [term 1 FOLLOWER]: Leader election won for term 1
08:07:35.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:35.008571 8842 raft_consensus.cc:697] T 00000000000000000000000000000000 P 4283b8cd94454e13bb77279c6afdb8f2 [term 1 LEADER]: Becoming Leader. State: Replica: 4283b8cd94454e13bb77279c6afdb8f2, State: Running, Role: LEADER
08:07:35.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:35.016055 8842 consensus_queue.cc:237] T 00000000000000000000000000000000 P 4283b8cd94454e13bb77279c6afdb8f2 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "62e54dd7499440acaa1b1c76403da552" member_type: VOTER last_known_addr { host: "127.8.87.254" port: 43115 } } peers { permanent_uuid: "4283b8cd94454e13bb77279c6afdb8f2" member_type: VOTER last_known_addr { host: "127.8.87.253" port: 46439 } } peers { permanent_uuid: "b0ae399fb47a454996698251cea1c12d" member_type: VOTER last_known_addr { host: "127.8.87.252" port: 38507 } }
08:07:35.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:35.029402 8842 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4283b8cd94454e13bb77279c6afdb8f2 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 4283b8cd94454e13bb77279c6afdb8f2. Latest consensus state: current_term: 1 leader_uuid: "4283b8cd94454e13bb77279c6afdb8f2" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "62e54dd7499440acaa1b1c76403da552" member_type: VOTER last_known_addr { host: "127.8.87.254" port: 43115 } } peers { permanent_uuid: "4283b8cd94454e13bb77279c6afdb8f2" member_type: VOTER last_known_addr { host: "127.8.87.253" port: 46439 } } peers { permanent_uuid: "b0ae399fb47a454996698251cea1c12d" member_type: VOTER last_known_addr { host: "127.8.87.252" port: 38507 } } }
08:07:35.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:35.030339 8842 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4283b8cd94454e13bb77279c6afdb8f2 [sys.catalog]: This master's current role is: LEADER
08:07:35.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:35.042239 8682 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.RequestConsensusVote -> {remote=127.8.87.254:43115, user_credentials={real_user=slave}} blocked reactor thread for 58516.1us
08:07:35.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:35.050578 8869 catalog_manager.cc:1568] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:07:35.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:35.050918 8869 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:07:35.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:35.054616 8871 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
08:07:35.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:35.077948 8871 catalog_manager.cc:1494] Initializing Kudu cluster ID...
08:07:35.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:35.109355 8794 raft_consensus.cc:1275] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d [term 1 FOLLOWER]: Refusing update from remote peer 4283b8cd94454e13bb77279c6afdb8f2: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
08:07:35.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:35.110965 8842 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 4283b8cd94454e13bb77279c6afdb8f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b0ae399fb47a454996698251cea1c12d" member_type: VOTER last_known_addr { host: "127.8.87.252" port: 38507 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
08:07:35.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:35.127578 8619 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 62e54dd7499440acaa1b1c76403da552 [term 1 FOLLOWER]: Refusing update from remote peer 4283b8cd94454e13bb77279c6afdb8f2: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
08:07:35.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:35.129446 8842 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 4283b8cd94454e13bb77279c6afdb8f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "62e54dd7499440acaa1b1c76403da552" member_type: VOTER last_known_addr { host: "127.8.87.254" port: 43115 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
08:07:35.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:35.159334 8843 sys_catalog.cc:455] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d [sys.catalog]: SysCatalogTable state changed. Reason: New leader 4283b8cd94454e13bb77279c6afdb8f2. Latest consensus state: current_term: 1 leader_uuid: "4283b8cd94454e13bb77279c6afdb8f2" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "62e54dd7499440acaa1b1c76403da552" member_type: VOTER last_known_addr { host: "127.8.87.254" port: 43115 } } peers { permanent_uuid: "4283b8cd94454e13bb77279c6afdb8f2" member_type: VOTER last_known_addr { host: "127.8.87.253" port: 46439 } } peers { permanent_uuid: "b0ae399fb47a454996698251cea1c12d" member_type: VOTER last_known_addr { host: "127.8.87.252" port: 38507 } } }
08:07:35.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:35.160199 8843 sys_catalog.cc:458] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d [sys.catalog]: This master's current role is: FOLLOWER
08:07:35.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:35.170346 8830 sys_catalog.cc:455] T 00000000000000000000000000000000 P 62e54dd7499440acaa1b1c76403da552 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 4283b8cd94454e13bb77279c6afdb8f2. Latest consensus state: current_term: 1 leader_uuid: "4283b8cd94454e13bb77279c6afdb8f2" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "62e54dd7499440acaa1b1c76403da552" member_type: VOTER last_known_addr { host: "127.8.87.254" port: 43115 } } peers { permanent_uuid: "4283b8cd94454e13bb77279c6afdb8f2" member_type: VOTER last_known_addr { host: "127.8.87.253" port: 46439 } } peers { permanent_uuid: "b0ae399fb47a454996698251cea1c12d" member_type: VOTER last_known_addr { host: "127.8.87.252" port: 38507 } } }
08:07:35.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:35.171375 8830 sys_catalog.cc:458] T 00000000000000000000000000000000 P 62e54dd7499440acaa1b1c76403da552 [sys.catalog]: This master's current role is: FOLLOWER
08:07:35.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:35.177903 8876 mvcc.cc:204] Tried to move back new op lower bound from 7213903278501720064 to 7213903278154756096. Current Snapshot: MvccSnapshot[applied={T|T < 7213903278501720064}]
08:07:35.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:35.203593 8843 sys_catalog.cc:455] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "4283b8cd94454e13bb77279c6afdb8f2" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "62e54dd7499440acaa1b1c76403da552" member_type: VOTER last_known_addr { host: "127.8.87.254" port: 43115 } } peers { permanent_uuid: "4283b8cd94454e13bb77279c6afdb8f2" member_type: VOTER last_known_addr { host: "127.8.87.253" port: 46439 } } peers { permanent_uuid: "b0ae399fb47a454996698251cea1c12d" member_type: VOTER last_known_addr { host: "127.8.87.252" port: 38507 } } }
08:07:35.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:35.204293 8843 sys_catalog.cc:458] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d [sys.catalog]: This master's current role is: FOLLOWER
08:07:35.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:35.209407 8871 catalog_manager.cc:1357] Generated new cluster ID: 8f877e344eb84cea8a008e14c9bc53c8
08:07:35.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:35.209692 8871 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
08:07:35.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:35.213346 8842 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4283b8cd94454e13bb77279c6afdb8f2 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "4283b8cd94454e13bb77279c6afdb8f2" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "62e54dd7499440acaa1b1c76403da552" member_type: VOTER last_known_addr { host: "127.8.87.254" port: 43115 } } peers { permanent_uuid: "4283b8cd94454e13bb77279c6afdb8f2" member_type: VOTER last_known_addr { host: "127.8.87.253" port: 46439 } } peers { permanent_uuid: "b0ae399fb47a454996698251cea1c12d" member_type: VOTER last_known_addr { host: "127.8.87.252" port: 38507 } } }
08:07:35.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:35.214017 8842 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4283b8cd94454e13bb77279c6afdb8f2 [sys.catalog]: This master's current role is: LEADER
08:07:35.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:35.214749 8842 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4283b8cd94454e13bb77279c6afdb8f2 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "4283b8cd94454e13bb77279c6afdb8f2" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "62e54dd7499440acaa1b1c76403da552" member_type: VOTER last_known_addr { host: "127.8.87.254" port: 43115 } } peers { permanent_uuid: "4283b8cd94454e13bb77279c6afdb8f2" member_type: VOTER last_known_addr { host: "127.8.87.253" port: 46439 } } peers { permanent_uuid: "b0ae399fb47a454996698251cea1c12d" member_type: VOTER last_known_addr { host: "127.8.87.252" port: 38507 } } }
08:07:35.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:35.215325 8842 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4283b8cd94454e13bb77279c6afdb8f2 [sys.catalog]: This master's current role is: LEADER
08:07:35.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:35.233758 8830 sys_catalog.cc:455] T 00000000000000000000000000000000 P 62e54dd7499440acaa1b1c76403da552 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "4283b8cd94454e13bb77279c6afdb8f2" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "62e54dd7499440acaa1b1c76403da552" member_type: VOTER last_known_addr { host: "127.8.87.254" port: 43115 } } peers { permanent_uuid: "4283b8cd94454e13bb77279c6afdb8f2" member_type: VOTER last_known_addr { host: "127.8.87.253" port: 46439 } } peers { permanent_uuid: "b0ae399fb47a454996698251cea1c12d" member_type: VOTER last_known_addr { host: "127.8.87.252" port: 38507 } } }
08:07:35.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:35.234469 8830 sys_catalog.cc:458] T 00000000000000000000000000000000 P 62e54dd7499440acaa1b1c76403da552 [sys.catalog]: This master's current role is: FOLLOWER
08:07:35.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:35.262245 8871 catalog_manager.cc:1380] Generated new certificate authority record
08:07:35.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:35.271533 8871 catalog_manager.cc:1514] Loading token signing keys...
08:07:35.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:35.330677 8871 catalog_manager.cc:6022] T 00000000000000000000000000000000 P 4283b8cd94454e13bb77279c6afdb8f2: Generated new TSK 0
08:07:35.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:35.331888 8871 catalog_manager.cc:1524] Initializing in-progress tserver states...
08:07:35.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.8.87.193@KRBTEST.COM; defaulting to no policy
08:07:35.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:07:35.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.8.87.193@KRBTEST.COM" created.
08:07:35.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:07:35.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.8.87.193 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/krb5kdc/kudu.keytab.
08:07:35.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.8.87.193 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/krb5kdc/kudu.keytab.
08:07:35.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.8.87.193@KRBTEST.COM; defaulting to no policy
08:07:35.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:07:35.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.8.87.193@KRBTEST.COM" created.
08:07:35.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:07:35.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.8.87.193 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/krb5kdc/kudu.keytab.
08:07:35.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.8.87.193 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/krb5kdc/kudu.keytab.
08:07:35.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:35.485810 8543 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskcrgkbR/build/tsan/bin/kudu
08:07:35.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskcrgkbR/build/tsan/bin/kudu
08:07:35.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/ts-0/wal
08:07:35.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/ts-0/data
08:07:35.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:07:35.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:07:35.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:07:35.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:07:35.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:07:35.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:07:35.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/ts-0/logs
08:07:35.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/ts-0/data/info.pb
08:07:35.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:07:35.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:07:35.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:07:35.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:07:35.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:07:35.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:07:35.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:07:35.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:07:35.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
08:07:35.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:07:35.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.87.193:0
08:07:35.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.8.87.193
08:07:35.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.87.193
08:07:35.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:07:35.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.8.87.254:43115,127.8.87.253:46439,127.8.87.252:38507
08:07:35.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/krb5kdc/kudu.keytab
08:07:35.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.8.87.193
08:07:35.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:07:35.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
08:07:35.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
08:07:35.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
08:07:35.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.87.212:46335
08:07:35.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:07:35.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:07:35.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:07:35.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false with env {KRB5CCNAME=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
08:07:35.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:35.885989 8851 catalog_manager.cc:1269] Loaded cluster ID: 8f877e344eb84cea8a008e14c9bc53c8
08:07:35.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:35.892638 8851 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 62e54dd7499440acaa1b1c76403da552: loading cluster ID for follower catalog manager: success
08:07:35.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:35.911172 8851 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 62e54dd7499440acaa1b1c76403da552: acquiring CA information for follower catalog manager: success
08:07:35.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:35.923200 8851 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 62e54dd7499440acaa1b1c76403da552: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
08:07:36.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:36.056998 8869 catalog_manager.cc:1269] Loaded cluster ID: 8f877e344eb84cea8a008e14c9bc53c8
08:07:36.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:36.057597 8869 catalog_manager.cc:1562] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d: loading cluster ID for follower catalog manager: success
08:07:36.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:36.065799 8869 catalog_manager.cc:1584] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d: acquiring CA information for follower catalog manager: success
08:07:36.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:36.082231 8869 catalog_manager.cc:1612] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
08:07:36.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:36.512766 8887 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:07:36.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:36.513703 8887 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:07:36.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:36.514935 8887 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:07:36.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:36.578578 8887 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:07:36.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:36.580436 8887 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.8.87.193
08:07:36.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:36.647193 8887 tablet_server_runner.cc:78] Tablet server non-default flags:
08:07:36.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:07:36.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.87.212:46335
08:07:36.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:07:36.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:07:36.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/ts-0/data
08:07:36.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/ts-0/wal
08:07:36.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/krb5kdc/kudu.keytab
08:07:36.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.8.87.193
08:07:36.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:07:36.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:07:36.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.87.193:0
08:07:36.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:07:36.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:07:36.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
08:07:36.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
08:07:36.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:07:36.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:07:36.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/ts-0/data/info.pb
08:07:36.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.87.193
08:07:36.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:07:36.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.8.87.254:43115,127.8.87.253:46439,127.8.87.252:38507
08:07:36.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:07:36.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:07:36.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:07:36.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:07:36.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:07:36.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.8.87.193
08:07:36.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/ts-0/logs
08:07:36.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:07:36.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:07:36.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:07:36.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
08:07:36.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:07:36.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 18b1930e830609248ee2e26d6216d073ff40ac5d
08:07:36.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:07:36.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 23 Oct 2025 07:43:21 UTC on 5fd53c4cbb9d
08:07:36.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8677
08:07:36.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:07:36.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:36.651274 8887 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:07:36.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:36.652901 8887 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:07:36.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:36.684791 8899 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:07:38.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:36.685078 8898 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:07:38.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:38.076700 8897 debug-util.cc:398] Leaking SignalData structure 0x7b080002ee00 after lost signal to thread 8887
08:07:38.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:38.882279 8887 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 2.200s user 0.523s sys 0.909s
08:07:38.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:38.883453 8887 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 2.201s user 0.524s sys 0.909s
08:07:38.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:38.896011 8887 server_base.cc:1047] running on GCE node
08:07:38.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:38.896476 8905 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:07:38.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:38.897924 8887 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:07:38.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:38.900668 8887 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:07:38.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:38.902190 8887 hybrid_clock.cc:648] HybridClock initialized: now 1761206858902139 us; error 40 us; skew 500 ppm
08:07:38.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:38.903249 8887 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:07:38.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 23 08:07:38 dist-test-slave-wxqn krb5kdc[8555](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761206858, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.87.193@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:07:38.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:38.911871 8887 init.cc:377] Logged in from keytab as kudu/127.8.87.193@KRBTEST.COM (short username kudu)
08:07:38.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:38.919178 8887 webserver.cc:492] Webserver started at http://127.8.87.193:44081/ using document root <none> and password file <none>
08:07:38.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:38.920310 8887 fs_manager.cc:362] Metadata directory not provided
08:07:38.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:38.920589 8887 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:07:38.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:38.921156 8887 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
08:07:38.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:38.927636 8887 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/ts-0/data/instance:
08:07:38.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b89b7d622ef34a1cb032858329be126a"
08:07:38.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-23 08:07:38 on dist-test-slave-wxqn"
08:07:38.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:38.929054 8887 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/ts-0/wal/instance:
08:07:38.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b89b7d622ef34a1cb032858329be126a"
08:07:38.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-23 08:07:38 on dist-test-slave-wxqn"
08:07:38.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:38.941905 8887 fs_manager.cc:696] Time spent creating directory manager: real 0.012s user 0.008s sys 0.004s
08:07:38.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:38.949249 8911 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:07:38.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:38.950613 8887 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.000s sys 0.003s
08:07:38.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:38.951040 8887 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/ts-0/data,/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/ts-0/wal
08:07:38.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b89b7d622ef34a1cb032858329be126a"
08:07:38.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-23 08:07:38 on dist-test-slave-wxqn"
08:07:38.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:38.951493 8887 fs_report.cc:389] FS layout report
08:07:38.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:07:38.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/ts-0/wal
08:07:38.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/ts-0/wal
08:07:38.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/ts-0/data/data
08:07:38.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:07:38.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:07:38.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:07:38.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:07:38.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:07:38.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:07:38.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:07:38.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:07:38.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:07:38.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:07:38.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:07:39.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:39.020135 8887 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:07:39.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:39.028043 8887 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:07:39.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:39.028725 8887 kserver.cc:163] Server-wide thread pool size limit: 3276
08:07:39.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:39.032225 8887 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
08:07:39.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:39.038962 8887 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
08:07:39.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:39.039230 8887 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
08:07:39.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:39.039530 8887 ts_tablet_manager.cc:616] Registered 0 tablets
08:07:39.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:39.039744 8887 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
08:07:39.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:39.244112 8887 rpc_server.cc:307] RPC server started. Bound to: 127.8.87.193:36301
08:07:39.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:39.244232 9023 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.8.87.193:36301 every 8 connection(s)
08:07:39.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:39.246592 8887 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/ts-0/data/info.pb
08:07:39.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:39.250260 8543 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskcrgkbR/build/tsan/bin/kudu as pid 8887
08:07:39.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:39.250818 8543 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/ts-0/wal/instance
08:07:39.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 23 08:07:39 dist-test-slave-wxqn krb5kdc[8555](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761206859, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.87.254@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:07:39.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:39.291090 8588 init.cc:295] Successfully reacquired a new kerberos TGT
08:07:39.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.8.87.194@KRBTEST.COM; defaulting to no policy
08:07:39.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:07:39.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.8.87.194@KRBTEST.COM" created.
08:07:39.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 23 08:07:39 dist-test-slave-wxqn krb5kdc[8555](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761206858, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.87.193@KRBTEST.COM for kudu/127.8.87.253@KRBTEST.COM
08:07:39.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 23 08:07:39 dist-test-slave-wxqn krb5kdc[8555](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761206858, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.87.193@KRBTEST.COM for kudu/127.8.87.254@KRBTEST.COM
08:07:39.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 23 08:07:39 dist-test-slave-wxqn krb5kdc[8555](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761206858, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.87.193@KRBTEST.COM for kudu/127.8.87.252@KRBTEST.COM
08:07:39.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:39.337024 9024 heartbeater.cc:344] Connected to a master server at 127.8.87.253:46439
08:07:39.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:39.337535 9024 heartbeater.cc:461] Registering TS with master...
08:07:39.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:39.338888 9024 heartbeater.cc:507] Master 127.8.87.253:46439 requested a full tablet report, sending...
08:07:39.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:39.342111 9029 heartbeater.cc:344] Connected to a master server at 127.8.87.254:43115
08:07:39.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:39.342443 9029 heartbeater.cc:461] Registering TS with master...
08:07:39.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:39.342962 8697 ts_manager.cc:194] Registered new tserver with Master: b89b7d622ef34a1cb032858329be126a (127.8.87.193:36301)
08:07:39.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:39.343346 9029 heartbeater.cc:507] Master 127.8.87.254:43115 requested a full tablet report, sending...
08:07:39.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:39.344890 9025 heartbeater.cc:344] Connected to a master server at 127.8.87.252:38507
08:07:39.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:39.345714 9025 heartbeater.cc:461] Registering TS with master...
08:07:39.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:39.346585 9025 heartbeater.cc:507] Master 127.8.87.252:38507 requested a full tablet report, sending...
08:07:39.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:39.347548 8609 ts_manager.cc:194] Registered new tserver with Master: b89b7d622ef34a1cb032858329be126a (127.8.87.193:36301)
08:07:39.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:39.348351 8697 master_service.cc:502] Signed X509 certificate for tserver {username='kudu', principal='kudu/127.8.87.193@KRBTEST.COM'} at 127.8.87.193:57531
08:07:39.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:39.349304 8784 ts_manager.cc:194] Registered new tserver with Master: b89b7d622ef34a1cb032858329be126a (127.8.87.193:36301)
08:07:39.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:07:39.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.8.87.194 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/krb5kdc/kudu.keytab.
08:07:39.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.8.87.194 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/krb5kdc/kudu.keytab.
08:07:39.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.8.87.194@KRBTEST.COM; defaulting to no policy
08:07:39.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:07:39.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.8.87.194@KRBTEST.COM" created.
08:07:39.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:07:39.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.8.87.194 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/krb5kdc/kudu.keytab.
08:07:39.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.8.87.194 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/krb5kdc/kudu.keytab.
08:07:39.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:39.397819 8543 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskcrgkbR/build/tsan/bin/kudu
08:07:39.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskcrgkbR/build/tsan/bin/kudu
08:07:39.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/ts-1/wal
08:07:39.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/ts-1/data
08:07:39.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:07:39.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:07:39.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:07:39.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:07:39.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:07:39.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:07:39.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/ts-1/logs
08:07:39.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/ts-1/data/info.pb
08:07:39.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:07:39.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:07:39.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:07:39.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:07:39.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:07:39.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:07:39.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:07:39.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:07:39.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
08:07:39.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:07:39.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.87.194:0
08:07:39.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.8.87.194
08:07:39.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.87.194
08:07:39.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:07:39.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.8.87.254:43115,127.8.87.253:46439,127.8.87.252:38507
08:07:39.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/krb5kdc/kudu.keytab
08:07:39.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.8.87.194
08:07:39.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:07:39.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
08:07:39.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
08:07:39.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
08:07:39.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.87.212:46335
08:07:39.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:07:39.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:07:39.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:07:39.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false with env {KRB5CCNAME=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
08:07:39.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:39.764238 9039 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:07:39.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:39.764767 9039 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:07:39.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:39.765317 9039 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:07:39.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:39.798002 9039 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:07:39.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:39.799094 9039 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.8.87.194
08:07:39.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:39.839175 9039 tablet_server_runner.cc:78] Tablet server non-default flags:
08:07:39.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:07:39.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.87.212:46335
08:07:39.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:07:39.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:07:39.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/ts-1/data
08:07:39.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/ts-1/wal
08:07:39.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/krb5kdc/kudu.keytab
08:07:39.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.8.87.194
08:07:39.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:07:39.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:07:39.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.87.194:0
08:07:39.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:07:39.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:07:39.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
08:07:39.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
08:07:39.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:07:39.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:07:39.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/ts-1/data/info.pb
08:07:39.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.87.194
08:07:39.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:07:39.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.8.87.254:43115,127.8.87.253:46439,127.8.87.252:38507
08:07:39.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:07:39.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:07:39.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:07:39.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:07:39.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:07:39.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.8.87.194
08:07:39.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/ts-1/logs
08:07:39.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:07:39.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:07:39.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:07:39.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
08:07:39.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:07:39.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 18b1930e830609248ee2e26d6216d073ff40ac5d
08:07:39.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:07:39.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 23 Oct 2025 07:43:21 UTC on 5fd53c4cbb9d
08:07:39.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8677
08:07:39.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:07:39.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:39.840330 9039 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:07:39.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:39.841573 9039 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:07:39.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:39.854628 9047 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:07:40.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:40.352797 9024 heartbeater.cc:499] Master 127.8.87.253:46439 was elected leader, sending a full tablet report...
08:07:41.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:41.257181 9046 debug-util.cc:398] Leaking SignalData structure 0x7b080002ee00 after lost signal to thread 9039
08:07:41.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:41.710940 9039 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.855s user 0.621s sys 1.156s
08:07:41.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:41.711364 9039 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.855s user 0.621s sys 1.156s
08:07:41.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:39.856350 9048 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:07:41.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:41.713326 9049 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1855 milliseconds
08:07:41.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:41.713954 9039 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
08:07:41.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:41.714016 9050 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:07:41.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:41.718926 9039 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:07:41.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:41.721056 9039 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:07:41.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:41.722451 9039 hybrid_clock.cc:648] HybridClock initialized: now 1761206861722402 us; error 57 us; skew 500 ppm
08:07:41.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:41.723356 9039 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:07:41.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 23 08:07:41 dist-test-slave-wxqn krb5kdc[8555](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761206861, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.87.194@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:07:41.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:41.729612 9039 init.cc:377] Logged in from keytab as kudu/127.8.87.194@KRBTEST.COM (short username kudu)
08:07:41.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:41.735472 9039 webserver.cc:492] Webserver started at http://127.8.87.194:33445/ using document root <none> and password file <none>
08:07:41.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:41.736322 9039 fs_manager.cc:362] Metadata directory not provided
08:07:41.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:41.736559 9039 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:07:41.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:41.736953 9039 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
08:07:41.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:41.740980 9039 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/ts-1/data/instance:
08:07:41.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0b8b4a4be8314be4b54168e19654fe19"
08:07:41.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-23 08:07:41 on dist-test-slave-wxqn"
08:07:41.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:41.742007 9039 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/ts-1/wal/instance:
08:07:41.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0b8b4a4be8314be4b54168e19654fe19"
08:07:41.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-23 08:07:41 on dist-test-slave-wxqn"
08:07:41.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:41.751094 9039 fs_manager.cc:696] Time spent creating directory manager: real 0.009s user 0.004s sys 0.005s
08:07:41.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:41.757318 9060 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:07:41.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:41.758435 9039 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
08:07:41.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:41.758751 9039 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/ts-1/data,/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/ts-1/wal
08:07:41.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0b8b4a4be8314be4b54168e19654fe19"
08:07:41.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-23 08:07:41 on dist-test-slave-wxqn"
08:07:41.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:41.759074 9039 fs_report.cc:389] FS layout report
08:07:41.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:07:41.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/ts-1/wal
08:07:41.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/ts-1/wal
08:07:41.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/ts-1/data/data
08:07:41.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:07:41.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:07:41.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:07:41.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:07:41.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:07:41.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:07:41.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:07:41.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:07:41.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:07:41.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:07:41.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:07:41.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:41.835232 9039 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:07:41.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:41.843578 9039 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:07:41.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:41.843971 9039 kserver.cc:163] Server-wide thread pool size limit: 3276
08:07:41.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:41.847102 9039 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
08:07:41.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:41.851512 9039 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
08:07:41.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:41.851732 9039 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
08:07:41.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:41.851920 9039 ts_tablet_manager.cc:616] Registered 0 tablets
08:07:41.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:41.852052 9039 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
08:07:41.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:41.925828 8389 debug-util.cc:398] Leaking SignalData structure 0x7b08000b6a80 after lost signal to thread 8264
08:07:41.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:41.927062 8389 debug-util.cc:398] Leaking SignalData structure 0x7b08000b6b00 after lost signal to thread 8392
08:07:41.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 23 08:07:41 dist-test-slave-wxqn krb5kdc[8555](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761206861, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.87.253@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:07:41.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:41.997627 8675 init.cc:295] Successfully reacquired a new kerberos TGT
08:07:42.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:42.059069 9039 rpc_server.cc:307] RPC server started. Bound to: 127.8.87.194:43889
08:07:42.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:42.059279 9173 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.8.87.194:43889 every 8 connection(s)
08:07:42.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:42.062559 9039 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/ts-1/data/info.pb
08:07:42.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:42.069206 8543 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskcrgkbR/build/tsan/bin/kudu as pid 9039
08:07:42.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:42.069702 8543 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/ts-1/wal/instance
08:07:42.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.8.87.195@KRBTEST.COM; defaulting to no policy
08:07:42.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:07:42.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.8.87.195@KRBTEST.COM" created.
08:07:42.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 23 08:07:42 dist-test-slave-wxqn krb5kdc[8555](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761206861, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.87.194@KRBTEST.COM for kudu/127.8.87.253@KRBTEST.COM
08:07:42.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 23 08:07:42 dist-test-slave-wxqn krb5kdc[8555](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761206861, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.87.194@KRBTEST.COM for kudu/127.8.87.252@KRBTEST.COM
08:07:42.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 23 08:07:42 dist-test-slave-wxqn krb5kdc[8555](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761206861, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.87.194@KRBTEST.COM for kudu/127.8.87.254@KRBTEST.COM
08:07:42.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:42.164985 9176 heartbeater.cc:344] Connected to a master server at 127.8.87.254:43115
08:07:42.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:42.165341 9175 heartbeater.cc:344] Connected to a master server at 127.8.87.252:38507
08:07:42.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:42.165520 9176 heartbeater.cc:461] Registering TS with master...
08:07:42.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:42.165694 9175 heartbeater.cc:461] Registering TS with master...
08:07:42.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:42.166054 9174 heartbeater.cc:344] Connected to a master server at 127.8.87.253:46439
08:07:42.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:42.166356 9174 heartbeater.cc:461] Registering TS with master...
08:07:42.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:42.168519 9176 heartbeater.cc:507] Master 127.8.87.254:43115 requested a full tablet report, sending...
08:07:42.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:42.168668 9174 heartbeater.cc:507] Master 127.8.87.253:46439 requested a full tablet report, sending...
08:07:42.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:42.174701 8609 ts_manager.cc:194] Registered new tserver with Master: 0b8b4a4be8314be4b54168e19654fe19 (127.8.87.194:43889)
08:07:42.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:42.178791 9175 heartbeater.cc:507] Master 127.8.87.252:38507 requested a full tablet report, sending...
08:07:42.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:42.178889 8697 ts_manager.cc:194] Registered new tserver with Master: 0b8b4a4be8314be4b54168e19654fe19 (127.8.87.194:43889)
08:07:42.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:42.181272 8697 master_service.cc:502] Signed X509 certificate for tserver {username='kudu', principal='kudu/127.8.87.194@KRBTEST.COM'} at 127.8.87.194:34183
08:07:42.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:42.182071 8784 ts_manager.cc:194] Registered new tserver with Master: 0b8b4a4be8314be4b54168e19654fe19 (127.8.87.194:43889)
08:07:42.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:07:42.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.8.87.195 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/krb5kdc/kudu.keytab.
08:07:42.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.8.87.195 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/krb5kdc/kudu.keytab.
08:07:42.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:42.188655 8543 mini_kdc.cc:279] Time spent creating service keytab for kudu/127.8.87.195: real 0.104s user 0.002s sys 0.014s
08:07:42.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.8.87.195@KRBTEST.COM; defaulting to no policy
08:07:42.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:07:42.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.8.87.195@KRBTEST.COM" created.
08:07:42.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:07:42.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.8.87.195 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/krb5kdc/kudu.keytab.
08:07:42.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.8.87.195 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/krb5kdc/kudu.keytab.
08:07:42.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:42.240680 8543 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskcrgkbR/build/tsan/bin/kudu
08:07:42.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskcrgkbR/build/tsan/bin/kudu
08:07:42.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/ts-2/wal
08:07:42.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/ts-2/data
08:07:42.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:07:42.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:07:42.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:07:42.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:07:42.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:07:42.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:07:42.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/ts-2/logs
08:07:42.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/ts-2/data/info.pb
08:07:42.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:07:42.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:07:42.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:07:42.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:07:42.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:07:42.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:07:42.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:07:42.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:07:42.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
08:07:42.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:07:42.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.87.195:0
08:07:42.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.8.87.195
08:07:42.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.87.195
08:07:42.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:07:42.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.8.87.254:43115,127.8.87.253:46439,127.8.87.252:38507
08:07:42.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/krb5kdc/kudu.keytab
08:07:42.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.8.87.195
08:07:42.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:07:42.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
08:07:42.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
08:07:42.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
08:07:42.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.87.212:46335
08:07:42.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:07:42.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:07:42.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:07:42.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false with env {KRB5CCNAME=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
08:07:42.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:42.735330 9188 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:07:42.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:42.735870 9188 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:07:42.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:42.736430 9188 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:07:42.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:42.771427 9188 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:07:42.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:42.772485 9188 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.8.87.195
08:07:42.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:42.810020 9188 tablet_server_runner.cc:78] Tablet server non-default flags:
08:07:42.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:07:42.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.87.212:46335
08:07:42.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:07:42.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:07:42.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/ts-2/data
08:07:42.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/ts-2/wal
08:07:42.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/krb5kdc/kudu.keytab
08:07:42.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.8.87.195
08:07:42.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:07:42.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:07:42.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.87.195:0
08:07:42.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:07:42.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:07:42.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
08:07:42.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
08:07:42.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:07:42.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:07:42.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/ts-2/data/info.pb
08:07:42.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.87.195
08:07:42.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:07:42.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.8.87.254:43115,127.8.87.253:46439,127.8.87.252:38507
08:07:42.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:07:42.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:07:42.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:07:42.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:07:42.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:07:42.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.8.87.195
08:07:42.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/ts-2/logs
08:07:42.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:07:42.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:07:42.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:07:42.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
08:07:42.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:07:42.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 18b1930e830609248ee2e26d6216d073ff40ac5d
08:07:42.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:07:42.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 23 Oct 2025 07:43:21 UTC on 5fd53c4cbb9d
08:07:42.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8677
08:07:42.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:07:42.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:42.811668 9188 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:07:42.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:42.812868 9188 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:07:42.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:42.825778 9196 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:07:43.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:43.185711 9174 heartbeater.cc:499] Master 127.8.87.253:46439 was elected leader, sending a full tablet report...
08:07:44.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:42.829658 9199 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:07:44.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:42.829514 9197 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:07:44.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:44.098907 9188 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
08:07:44.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:44.099172 9198 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
08:07:44.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:44.104168 9188 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:07:44.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:44.107339 9188 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:07:44.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:44.108971 9188 hybrid_clock.cc:648] HybridClock initialized: now 1761206864108867 us; error 90 us; skew 500 ppm
08:07:44.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:44.110267 9188 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:07:44.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 23 08:07:44 dist-test-slave-wxqn krb5kdc[8555](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761206864, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.87.195@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:07:44.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:44.120743 9188 init.cc:377] Logged in from keytab as kudu/127.8.87.195@KRBTEST.COM (short username kudu)
08:07:44.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:44.136042 9188 webserver.cc:492] Webserver started at http://127.8.87.195:42143/ using document root <none> and password file <none>
08:07:44.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:44.137477 9188 fs_manager.cc:362] Metadata directory not provided
08:07:44.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:44.137768 9188 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:07:44.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:44.138423 9188 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
08:07:44.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:44.146210 9188 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/ts-2/data/instance:
08:07:44.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2c485d5a4b9f4ba6926df4b0102eed27"
08:07:44.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-23 08:07:44 on dist-test-slave-wxqn"
08:07:44.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:44.147893 9188 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/ts-2/wal/instance:
08:07:44.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2c485d5a4b9f4ba6926df4b0102eed27"
08:07:44.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-23 08:07:44 on dist-test-slave-wxqn"
08:07:44.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:44.158807 9188 fs_manager.cc:696] Time spent creating directory manager: real 0.010s user 0.005s sys 0.004s
08:07:44.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:44.165289 9208 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:07:44.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:44.166420 9188 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.002s
08:07:44.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:44.166724 9188 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/ts-2/data,/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/ts-2/wal
08:07:44.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2c485d5a4b9f4ba6926df4b0102eed27"
08:07:44.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-23 08:07:44 on dist-test-slave-wxqn"
08:07:44.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:44.167027 9188 fs_report.cc:389] FS layout report
08:07:44.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:07:44.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/ts-2/wal
08:07:44.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/ts-2/wal
08:07:44.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/ts-2/data/data
08:07:44.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:07:44.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:07:44.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:07:44.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:07:44.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:07:44.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:07:44.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:07:44.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:07:44.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:07:44.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:07:44.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:07:44.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 23 08:07:44 dist-test-slave-wxqn krb5kdc[8555](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761206864, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.87.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:07:44.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:44.270996 8763 init.cc:295] Successfully reacquired a new kerberos TGT
08:07:44.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:44.271476 9188 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:07:44.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:44.280217 9188 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:07:44.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:44.281037 9188 kserver.cc:163] Server-wide thread pool size limit: 3276
08:07:44.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:44.287333 9188 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
08:07:44.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:44.296571 9188 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
08:07:44.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:44.296830 9188 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.001s sys 0.000s
08:07:44.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:44.297125 9188 ts_tablet_manager.cc:616] Registered 0 tablets
08:07:44.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:44.297299 9188 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
08:07:44.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:44.612257 9188 rpc_server.cc:307] RPC server started. Bound to: 127.8.87.195:36087
08:07:44.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:44.612324 9320 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.8.87.195:36087 every 8 connection(s)
08:07:44.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:44.615983 9188 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/ts-2/data/info.pb
08:07:44.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:44.625463 8543 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskcrgkbR/build/tsan/bin/kudu as pid 9188
08:07:44.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:44.626036 8543 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/ts-2/wal/instance
08:07:44.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 23 08:07:44 dist-test-slave-wxqn krb5kdc[8555](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761206864, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.87.195@KRBTEST.COM for kudu/127.8.87.252@KRBTEST.COM
08:07:44.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:44.704916 9322 heartbeater.cc:344] Connected to a master server at 127.8.87.252:38507
08:07:44.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:44.705880 9322 heartbeater.cc:461] Registering TS with master...
08:07:44.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 23 08:07:44 dist-test-slave-wxqn krb5kdc[8555](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761206864, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.87.195@KRBTEST.COM for kudu/127.8.87.253@KRBTEST.COM
08:07:44.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:44.709798 9322 heartbeater.cc:507] Master 127.8.87.252:38507 requested a full tablet report, sending...
08:07:44.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:44.713220 8784 ts_manager.cc:194] Registered new tserver with Master: 2c485d5a4b9f4ba6926df4b0102eed27 (127.8.87.195:36087)
08:07:44.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 23 08:07:44 dist-test-slave-wxqn krb5kdc[8555](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761206864, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.87.195@KRBTEST.COM for kudu/127.8.87.254@KRBTEST.COM
08:07:44.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:44.723191 9321 heartbeater.cc:344] Connected to a master server at 127.8.87.253:46439
08:07:44.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:44.723583 9321 heartbeater.cc:461] Registering TS with master...
08:07:44.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:44.724548 9321 heartbeater.cc:507] Master 127.8.87.253:46439 requested a full tablet report, sending...
08:07:44.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:44.726857 8697 ts_manager.cc:194] Registered new tserver with Master: 2c485d5a4b9f4ba6926df4b0102eed27 (127.8.87.195:36087)
08:07:44.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:44.729157 8697 master_service.cc:502] Signed X509 certificate for tserver {username='kudu', principal='kudu/127.8.87.195@KRBTEST.COM'} at 127.8.87.195:41893
08:07:44.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:44.733851 9323 heartbeater.cc:344] Connected to a master server at 127.8.87.254:43115
08:07:44.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:44.734210 9323 heartbeater.cc:461] Registering TS with master...
08:07:44.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:44.734825 9323 heartbeater.cc:507] Master 127.8.87.254:43115 requested a full tablet report, sending...
08:07:44.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:44.737618 8609 ts_manager.cc:194] Registered new tserver with Master: 2c485d5a4b9f4ba6926df4b0102eed27 (127.8.87.195:36087)
08:07:44.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:44.747198 8543 external_mini_cluster.cc:949] 3 TS(s) registered with all masters
08:07:44.761 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:07:44.763 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
08:07:44.775 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
08:07:44.775 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
08:07:44.778 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.8.87.254"
port: 43115
}
bound_http_address {
host: "127.8.87.254"
port: 33423
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.8.87.253"
port: 46439
}
bound_http_address {
host: "127.8.87.253"
port: 44589
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.8.87.252"
port: 38507
}
bound_http_address {
host: "127.8.87.252"
port: 40629
}
}
}
08:07:44.779 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
08:07:44.780 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.8.87.193"
port: 36301
}
bound_http_address {
host: "127.8.87.193"
port: 44081
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.8.87.194"
port: 43889
}
bound_http_address {
host: "127.8.87.194"
port: 33445
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.8.87.195"
port: 36087
}
bound_http_address {
host: "127.8.87.195"
port: 42143
}
}
}
08:07:44.781 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
08:07:44.782 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
08:07:44.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 23 08:07:44 dist-test-slave-wxqn krb5kdc[8555](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761206864, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:07:44.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
08:07:44.818 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:07:44.820 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/krb5kdc/krb5cc
08:07:44.825 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
08:07:44.826 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/krb5kdc/krb5cc
08:07:44.827 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
08:07:44.830 [DEBUG - Time-limited test] (SecurityContext.java:158) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
08:07:44.832 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
08:07:44.832 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
08:07:44.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 23 08:07:44 dist-test-slave-wxqn krb5kdc[8555](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761206864, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:07:44.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
08:07:44.864 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:07:44.864 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/krb5kdc/krb5cc
08:07:44.866 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
08:07:44.866 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.8.87.252:38507
08:07:44.868 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
08:07:44.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:44.869161 8543 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskcrgkbR/build/tsan/bin/kudu with pid 8742
08:07:44.893 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:07:44.893 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.8.87.254:43115
08:07:44.894 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
08:07:44.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:44.894403 8543 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskcrgkbR/build/tsan/bin/kudu with pid 8573
08:07:44.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:44.925086 8682 connection.cc:537] client connection to 127.8.87.254:43115 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
08:07:44.925 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:07:44.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:44.925496 8682 proxy.cc:239] Call had error, refreshing address and retrying: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107) [suppressed 6 similar messages]
08:07:44.925 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.8.87.253:46439
08:07:44.926 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
08:07:44.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:44.926558 8543 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskcrgkbR/build/tsan/bin/kudu with pid 8654
08:07:44.955 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:07:44.956 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.8.87.252:38507
08:07:44.959 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
08:07:44.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:44.960955 8543 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskcrgkbR/build/tsan/bin/kudu
08:07:44.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskcrgkbR/build/tsan/bin/kudu
08:07:44.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-2/wal
08:07:44.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-2/data
08:07:44.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:07:44.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:07:44.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:07:44.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:07:44.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:07:44.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:07:44.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-2/logs
08:07:44.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-2/data/info.pb
08:07:44.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:07:44.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:07:44.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:07:44.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:07:44.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:07:44.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:07:44.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:07:44.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:07:44.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:07:44.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:07:44.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:07:44.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:07:44.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.87.252:38507
08:07:44.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.87.252
08:07:44.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=40629
08:07:44.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/krb5kdc/kudu.keytab
08:07:44.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.8.87.252
08:07:44.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:07:44.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
08:07:44.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
08:07:44.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
08:07:44.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.87.212:46335
08:07:44.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:07:44.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:07:44.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:07:44.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:07:44.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.87.254:43115,127.8.87.253:46439,127.8.87.252:38507
08:07:44.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:07:44.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40533/jwks.json
08:07:44.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
08:07:44.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
08:07:45.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:45.189622 9176 heartbeater.cc:646] Failed to heartbeat to 127.8.87.254:43115 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.8.87.254:43115: connect: Connection refused (error 111)
08:07:45.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:45.280416 9337 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:07:45.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:45.281024 9337 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:07:45.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:45.281633 9337 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:07:45.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:45.314872 9337 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
08:07:45.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:45.315097 9337 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
08:07:45.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:45.315357 9337 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:07:45.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:45.315550 9337 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
08:07:45.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:45.315749 9337 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
08:07:45.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:45.315938 9337 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
08:07:45.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:45.316075 9337 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:40533/jwks.json
08:07:45.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:45.353873 9337 master_runner.cc:386] Master server non-default flags:
08:07:45.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:07:45.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.87.212:46335
08:07:45.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:07:45.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:07:45.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-2/data
08:07:45.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-2/wal
08:07:45.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
08:07:45.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:07:45.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.87.254:43115,127.8.87.253:46439,127.8.87.252:38507
08:07:45.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/krb5kdc/kudu.keytab
08:07:45.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.8.87.252
08:07:45.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:07:45.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:07:45.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:07:45.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.87.252:38507
08:07:45.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:07:45.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:07:45.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:07:45.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40533/jwks.json
08:07:45.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:07:45.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
08:07:45.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
08:07:45.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:07:45.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:07:45.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-2/data/info.pb
08:07:45.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.87.252
08:07:45.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=40629
08:07:45.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:07:45.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:07:45.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:07:45.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:07:45.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:07:45.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-2/logs
08:07:45.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:07:45.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:07:45.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:07:45.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:07:45.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:07:45.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 18b1930e830609248ee2e26d6216d073ff40ac5d
08:07:45.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:07:45.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 23 Oct 2025 07:43:21 UTC on 5fd53c4cbb9d
08:07:45.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8677
08:07:45.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:07:45.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:45.355103 9337 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:07:45.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:45.356088 9337 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:07:45.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:45.366916 9345 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:07:45.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:45.442015 9024 heartbeater.cc:646] Failed to heartbeat to 127.8.87.253:46439 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.8.87.253:46439: connect: Connection refused (error 111)
08:07:45.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:45.442015 9029 heartbeater.cc:646] Failed to heartbeat to 127.8.87.254:43115 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.8.87.254:43115: connect: Connection refused (error 111)
08:07:45.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:45.719305 9322 heartbeater.cc:646] Failed to heartbeat to 127.8.87.252:38507 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.8.87.252:38507: connect: Connection refused (error 111)
08:07:45.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:45.736531 9321 heartbeater.cc:499] Master 127.8.87.253:46439 was elected leader, sending a full tablet report...
08:07:46.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:46.770113 9344 debug-util.cc:398] Leaking SignalData structure 0x7b0800027080 after lost signal to thread 9337
08:07:46.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:45.370019 9346 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:07:46.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:46.821039 9344 kernel_stack_watchdog.cc:198] Thread 9337 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 401ms:
08:07:46.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
08:07:46.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
08:07:46.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
08:07:46.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
08:07:46.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:46.821427 9347 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection timed out after 1453 milliseconds
08:07:46.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:46.825709 9337 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.459s user 0.524s sys 0.922s
08:07:46.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:46.826109 9337 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.459s user 0.524s sys 0.922s
08:07:46.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:46.827666 9348 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:07:46.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:46.827703 9337 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
08:07:46.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:46.829038 9337 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:07:46.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:46.831290 9337 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:07:46.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:46.832651 9337 hybrid_clock.cc:648] HybridClock initialized: now 1761206866832602 us; error 43 us; skew 500 ppm
08:07:46.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:46.833474 9337 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:07:46.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 23 08:07:46 dist-test-slave-wxqn krb5kdc[8555](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761206866, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.87.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:07:46.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:46.839684 9337 init.cc:377] Logged in from keytab as kudu/127.8.87.252@KRBTEST.COM (short username kudu)
08:07:46.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:46.844944 9337 webserver.cc:492] Webserver started at http://127.8.87.252:40629/ using document root <none> and password file <none>
08:07:46.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:46.845922 9337 fs_manager.cc:362] Metadata directory not provided
08:07:46.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:46.846150 9337 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:07:46.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:46.853736 9337 fs_manager.cc:714] Time spent opening directory manager: real 0.005s user 0.002s sys 0.002s
08:07:46.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:46.858343 9366 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:07:46.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:46.859400 9337 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.005s sys 0.000s
08:07:46.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:46.859740 9337 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-2/data,/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-2/wal
08:07:46.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b0ae399fb47a454996698251cea1c12d"
08:07:46.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-23 08:07:34 on dist-test-slave-wxqn"
08:07:46.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:46.861848 9337 fs_report.cc:389] FS layout report
08:07:46.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:07:46.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-2/wal
08:07:46.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-2/wal
08:07:46.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-2/data/data
08:07:46.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:07:46.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:07:46.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:07:46.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:07:46.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:07:46.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:07:46.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:07:46.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:07:46.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:07:46.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:07:46.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:07:46.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:46.916815 9337 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:07:46.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:46.926667 9337 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:07:46.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:46.927275 9337 kserver.cc:163] Server-wide thread pool size limit: 3276
08:07:47.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:47.006438 9337 rpc_server.cc:307] RPC server started. Bound to: 127.8.87.252:38507
08:07:47.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:47.006503 9418 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.8.87.252:38507 every 8 connection(s)
08:07:47.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:47.009346 9337 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-2/data/info.pb
08:07:47.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:47.016978 8543 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskcrgkbR/build/tsan/bin/kudu as pid 9337
08:07:47.017 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:07:47.017 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.8.87.254:43115
08:07:47.018 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
08:07:47.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:47.019237 8543 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskcrgkbR/build/tsan/bin/kudu
08:07:47.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskcrgkbR/build/tsan/bin/kudu
08:07:47.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-0/wal
08:07:47.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-0/data
08:07:47.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:07:47.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:07:47.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:07:47.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:07:47.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:07:47.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:07:47.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-0/logs
08:07:47.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-0/data/info.pb
08:07:47.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:07:47.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:07:47.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:07:47.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:07:47.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:07:47.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:07:47.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:07:47.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:07:47.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:07:47.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:07:47.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:07:47.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:07:47.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.87.254:43115
08:07:47.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.87.254
08:07:47.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33423
08:07:47.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/krb5kdc/kudu.keytab
08:07:47.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.8.87.254
08:07:47.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:07:47.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
08:07:47.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
08:07:47.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
08:07:47.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.87.212:46335
08:07:47.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:07:47.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:07:47.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:07:47.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:07:47.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.87.254:43115,127.8.87.253:46439,127.8.87.252:38507
08:07:47.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:07:47.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40533/jwks.json
08:07:47.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
08:07:47.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
08:07:47.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:47.020695 9419 sys_catalog.cc:263] Verifying existing consensus state
08:07:47.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:47.027240 9419 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d: Bootstrap starting.
08:07:47.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:47.075481 9419 log.cc:826] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d: Log is configured to *not* fsync() on all Append() calls
08:07:47.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:47.091498 9419 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
08:07:47.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:47.092252 9419 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d: Bootstrap complete.
08:07:47.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:47.114943 9419 raft_consensus.cc:359] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "62e54dd7499440acaa1b1c76403da552" member_type: VOTER last_known_addr { host: "127.8.87.254" port: 43115 } } peers { permanent_uuid: "4283b8cd94454e13bb77279c6afdb8f2" member_type: VOTER last_known_addr { host: "127.8.87.253" port: 46439 } } peers { permanent_uuid: "b0ae399fb47a454996698251cea1c12d" member_type: VOTER last_known_addr { host: "127.8.87.252" port: 38507 } }
08:07:47.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:47.117143 9419 raft_consensus.cc:740] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: b0ae399fb47a454996698251cea1c12d, State: Initialized, Role: FOLLOWER
08:07:47.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:47.117945 9419 consensus_queue.cc:260] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "62e54dd7499440acaa1b1c76403da552" member_type: VOTER last_known_addr { host: "127.8.87.254" port: 43115 } } peers { permanent_uuid: "4283b8cd94454e13bb77279c6afdb8f2" member_type: VOTER last_known_addr { host: "127.8.87.253" port: 46439 } } peers { permanent_uuid: "b0ae399fb47a454996698251cea1c12d" member_type: VOTER last_known_addr { host: "127.8.87.252" port: 38507 } }
08:07:47.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:47.121158 9423 sys_catalog.cc:455] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "62e54dd7499440acaa1b1c76403da552" member_type: VOTER last_known_addr { host: "127.8.87.254" port: 43115 } } peers { permanent_uuid: "4283b8cd94454e13bb77279c6afdb8f2" member_type: VOTER last_known_addr { host: "127.8.87.253" port: 46439 } } peers { permanent_uuid: "b0ae399fb47a454996698251cea1c12d" member_type: VOTER last_known_addr { host: "127.8.87.252" port: 38507 } } }
08:07:47.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:47.122165 9423 sys_catalog.cc:458] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d [sys.catalog]: This master's current role is: FOLLOWER
08:07:47.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:47.123139 9419 sys_catalog.cc:565] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d [sys.catalog]: configured and running, proceeding with master startup.
08:07:47.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:47.153607 9434 catalog_manager.cc:1269] Loaded cluster ID: 8f877e344eb84cea8a008e14c9bc53c8
08:07:47.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:47.153898 9434 catalog_manager.cc:1562] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d: loading cluster ID for follower catalog manager: success
08:07:47.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:47.161552 9434 catalog_manager.cc:1584] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d: acquiring CA information for follower catalog manager: success
08:07:47.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:47.167299 9434 catalog_manager.cc:1612] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
08:07:47.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:47.149539 9337 master_runner.cc:186] Error getting master registration for 127.8.87.254:43115: Network error: Client connection negotiation failed: client connection to 127.8.87.254:43115: connect: Connection refused (error 111),
08:07:47.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:47.173228 9337 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
08:07:47.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:47.275689 9175 heartbeater.cc:344] Connected to a master server at 127.8.87.252:38507
08:07:47.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:47.282342 9384 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "0b8b4a4be8314be4b54168e19654fe19" instance_seqno: 1761206862002646) as {username='kudu', principal='kudu/127.8.87.194@KRBTEST.COM'} at 127.8.87.194:55647; Asking this server to re-register.
08:07:47.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:47.283979 9175 heartbeater.cc:461] Registering TS with master...
08:07:47.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:47.287380 9384 ts_manager.cc:194] Registered new tserver with Master: 0b8b4a4be8314be4b54168e19654fe19 (127.8.87.194:43889)
08:07:47.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:47.440537 9421 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:07:47.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:47.441313 9421 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:07:47.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:47.442363 9421 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:07:47.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:47.481633 9025 heartbeater.cc:344] Connected to a master server at 127.8.87.252:38507
08:07:47.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:47.484189 9384 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "b89b7d622ef34a1cb032858329be126a" instance_seqno: 1761206859197829) as {username='kudu', principal='kudu/127.8.87.193@KRBTEST.COM'} at 127.8.87.193:43613; Asking this server to re-register.
08:07:47.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:47.485548 9025 heartbeater.cc:461] Registering TS with master...
08:07:47.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:47.488255 9384 ts_manager.cc:194] Registered new tserver with Master: b89b7d622ef34a1cb032858329be126a (127.8.87.193:36301)
08:07:47.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:47.490469 9421 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
08:07:47.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:47.490782 9421 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
08:07:47.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:47.491266 9421 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:07:47.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:47.491644 9421 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
08:07:47.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:47.491994 9421 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
08:07:47.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:47.492321 9421 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
08:07:47.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:47.492602 9421 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:40533/jwks.json
08:07:47.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:47.543218 9421 master_runner.cc:386] Master server non-default flags:
08:07:47.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:07:47.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.87.212:46335
08:07:47.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:07:47.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:07:47.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-0/data
08:07:47.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-0/wal
08:07:47.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
08:07:47.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:07:47.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.87.254:43115,127.8.87.253:46439,127.8.87.252:38507
08:07:47.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/krb5kdc/kudu.keytab
08:07:47.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.8.87.254
08:07:47.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:07:47.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:07:47.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:07:47.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.87.254:43115
08:07:47.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:07:47.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:07:47.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:07:47.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40533/jwks.json
08:07:47.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:07:47.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
08:07:47.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
08:07:47.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:07:47.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:07:47.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-0/data/info.pb
08:07:47.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.87.254
08:07:47.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33423
08:07:47.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:07:47.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:07:47.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:07:47.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:07:47.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:07:47.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-0/logs
08:07:47.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:07:47.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:07:47.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:07:47.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:07:47.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:07:47.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 18b1930e830609248ee2e26d6216d073ff40ac5d
08:07:47.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:07:47.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 23 Oct 2025 07:43:21 UTC on 5fd53c4cbb9d
08:07:47.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8677
08:07:47.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:07:47.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:47.544354 9421 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:07:47.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:47.545325 9421 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:07:47.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:47.556411 9448 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:07:47.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:47.798799 9322 heartbeater.cc:344] Connected to a master server at 127.8.87.252:38507
08:07:47.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:47.803042 9384 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "2c485d5a4b9f4ba6926df4b0102eed27" instance_seqno: 1761206864566991) as {username='kudu', principal='kudu/127.8.87.195@KRBTEST.COM'} at 127.8.87.195:53715; Asking this server to re-register.
08:07:47.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:47.805789 9322 heartbeater.cc:461] Registering TS with master...
08:07:47.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:47.810374 9384 ts_manager.cc:194] Registered new tserver with Master: 2c485d5a4b9f4ba6926df4b0102eed27 (127.8.87.195:36087)
08:07:48.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:48.362654 9457 raft_consensus.cc:493] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:07:48.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:48.366508 9457 raft_consensus.cc:515] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "62e54dd7499440acaa1b1c76403da552" member_type: VOTER last_known_addr { host: "127.8.87.254" port: 43115 } } peers { permanent_uuid: "4283b8cd94454e13bb77279c6afdb8f2" member_type: VOTER last_known_addr { host: "127.8.87.253" port: 46439 } } peers { permanent_uuid: "b0ae399fb47a454996698251cea1c12d" member_type: VOTER last_known_addr { host: "127.8.87.252" port: 38507 } }
08:07:48.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:48.382484 9368 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.87.254:43115: connect: Connection refused (error 111)
08:07:48.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:48.389271 9457 leader_election.cc:290] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 62e54dd7499440acaa1b1c76403da552 (127.8.87.254:43115), 4283b8cd94454e13bb77279c6afdb8f2 (127.8.87.253:46439)
08:07:48.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:48.399467 9370 leader_election.cc:336] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 4283b8cd94454e13bb77279c6afdb8f2 (127.8.87.253:46439): Network error: Client connection negotiation failed: client connection to 127.8.87.253:46439: connect: Connection refused (error 111)
08:07:48.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:48.401885 9368 leader_election.cc:336] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 62e54dd7499440acaa1b1c76403da552 (127.8.87.254:43115): Network error: Client connection negotiation failed: client connection to 127.8.87.254:43115: connect: Connection refused (error 111)
08:07:48.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:48.402287 9368 leader_election.cc:304] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: b0ae399fb47a454996698251cea1c12d; no voters: 4283b8cd94454e13bb77279c6afdb8f2, 62e54dd7499440acaa1b1c76403da552
08:07:48.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:48.412878 9457 raft_consensus.cc:2749] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
08:07:48.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 23 08:07:48 dist-test-slave-wxqn krb5kdc[8555](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761206868, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.87.193@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:07:48.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:48.917394 8907 init.cc:295] Successfully reacquired a new kerberos TGT
08:07:49.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:48.959836 9447 debug-util.cc:398] Leaking SignalData structure 0x7b0800027080 after lost signal to thread 9421
08:07:49.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:49.063283 9447 kernel_stack_watchdog.cc:198] Thread 9421 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 397ms:
08:07:49.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
08:07:49.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
08:07:49.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
08:07:49.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
08:07:49.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:47.558135 9449 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:07:49.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:49.073360 9421 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.514s user 0.002s sys 0.005s
08:07:49.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:49.073726 9421 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.514s user 0.002s sys 0.005s
08:07:49.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:49.073982 9421 server_base.cc:1047] running on GCE node
08:07:49.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:49.074712 9451 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:07:49.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:49.075881 9421 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:07:49.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:49.078735 9421 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:07:49.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:49.080183 9421 hybrid_clock.cc:648] HybridClock initialized: now 1761206869080135 us; error 47 us; skew 500 ppm
08:07:49.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:49.081061 9421 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:07:49.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 23 08:07:49 dist-test-slave-wxqn krb5kdc[8555](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761206869, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.87.254@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:07:49.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:49.087890 9421 init.cc:377] Logged in from keytab as kudu/127.8.87.254@KRBTEST.COM (short username kudu)
08:07:49.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:49.093684 9421 webserver.cc:492] Webserver started at http://127.8.87.254:33423/ using document root <none> and password file <none>
08:07:49.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:49.094904 9421 fs_manager.cc:362] Metadata directory not provided
08:07:49.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:49.095180 9421 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:07:49.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:49.106081 9421 fs_manager.cc:714] Time spent opening directory manager: real 0.007s user 0.007s sys 0.000s
08:07:49.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:49.111564 9471 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:07:49.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:49.112733 9421 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.001s
08:07:49.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:49.113094 9421 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-0/data,/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-0/wal
08:07:49.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "62e54dd7499440acaa1b1c76403da552"
08:07:49.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-23 08:07:29 on dist-test-slave-wxqn"
08:07:49.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:49.115183 9421 fs_report.cc:389] FS layout report
08:07:49.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:07:49.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-0/wal
08:07:49.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-0/wal
08:07:49.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-0/data/data
08:07:49.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:07:49.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:07:49.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:07:49.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:07:49.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:07:49.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:07:49.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:07:49.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:07:49.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:07:49.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:07:49.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:07:49.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:49.181306 9421 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:07:49.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:49.187677 9421 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:07:49.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:49.188170 9421 kserver.cc:163] Server-wide thread pool size limit: 3276
08:07:49.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:49.282938 9421 rpc_server.cc:307] RPC server started. Bound to: 127.8.87.254:43115
08:07:49.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:49.283058 9523 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.8.87.254:43115 every 8 connection(s)
08:07:49.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:49.286288 9421 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-0/data/info.pb
08:07:49.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:49.290972 8543 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskcrgkbR/build/tsan/bin/kudu as pid 9421
08:07:49.292 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:07:49.292 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.8.87.253:46439
08:07:49.292 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
08:07:49.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:49.293689 8543 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskcrgkbR/build/tsan/bin/kudu
08:07:49.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskcrgkbR/build/tsan/bin/kudu
08:07:49.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-1/wal
08:07:49.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-1/data
08:07:49.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:07:49.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:07:49.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:07:49.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:07:49.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:07:49.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:07:49.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-1/logs
08:07:49.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-1/data/info.pb
08:07:49.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:07:49.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:07:49.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:07:49.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:07:49.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:07:49.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:07:49.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:07:49.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:07:49.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:07:49.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:07:49.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:07:49.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:07:49.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.87.253:46439
08:07:49.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.87.253
08:07:49.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44589
08:07:49.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/krb5kdc/kudu.keytab
08:07:49.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.8.87.253
08:07:49.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:07:49.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
08:07:49.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
08:07:49.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
08:07:49.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.87.212:46335
08:07:49.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:07:49.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:07:49.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:07:49.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:07:49.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.87.254:43115,127.8.87.253:46439,127.8.87.252:38507
08:07:49.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:07:49.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40533/jwks.json
08:07:49.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
08:07:49.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
08:07:49.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:49.308780 9524 sys_catalog.cc:263] Verifying existing consensus state
08:07:49.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:49.317391 9524 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 62e54dd7499440acaa1b1c76403da552: Bootstrap starting.
08:07:49.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:49.374146 9524 log.cc:826] T 00000000000000000000000000000000 P 62e54dd7499440acaa1b1c76403da552: Log is configured to *not* fsync() on all Append() calls
08:07:49.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:49.404361 9524 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 62e54dd7499440acaa1b1c76403da552: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
08:07:49.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:49.425535 9524 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 62e54dd7499440acaa1b1c76403da552: Bootstrap complete.
08:07:49.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:49.468271 9524 raft_consensus.cc:359] T 00000000000000000000000000000000 P 62e54dd7499440acaa1b1c76403da552 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "62e54dd7499440acaa1b1c76403da552" member_type: VOTER last_known_addr { host: "127.8.87.254" port: 43115 } } peers { permanent_uuid: "4283b8cd94454e13bb77279c6afdb8f2" member_type: VOTER last_known_addr { host: "127.8.87.253" port: 46439 } } peers { permanent_uuid: "b0ae399fb47a454996698251cea1c12d" member_type: VOTER last_known_addr { host: "127.8.87.252" port: 38507 } }
08:07:49.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:49.471884 9524 raft_consensus.cc:740] T 00000000000000000000000000000000 P 62e54dd7499440acaa1b1c76403da552 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 62e54dd7499440acaa1b1c76403da552, State: Initialized, Role: FOLLOWER
08:07:49.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:49.473130 9524 consensus_queue.cc:260] T 00000000000000000000000000000000 P 62e54dd7499440acaa1b1c76403da552 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "62e54dd7499440acaa1b1c76403da552" member_type: VOTER last_known_addr { host: "127.8.87.254" port: 43115 } } peers { permanent_uuid: "4283b8cd94454e13bb77279c6afdb8f2" member_type: VOTER last_known_addr { host: "127.8.87.253" port: 46439 } } peers { permanent_uuid: "b0ae399fb47a454996698251cea1c12d" member_type: VOTER last_known_addr { host: "127.8.87.252" port: 38507 } }
08:07:49.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:49.476114 9529 sys_catalog.cc:455] T 00000000000000000000000000000000 P 62e54dd7499440acaa1b1c76403da552 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "62e54dd7499440acaa1b1c76403da552" member_type: VOTER last_known_addr { host: "127.8.87.254" port: 43115 } } peers { permanent_uuid: "4283b8cd94454e13bb77279c6afdb8f2" member_type: VOTER last_known_addr { host: "127.8.87.253" port: 46439 } } peers { permanent_uuid: "b0ae399fb47a454996698251cea1c12d" member_type: VOTER last_known_addr { host: "127.8.87.252" port: 38507 } } }
08:07:49.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:49.477085 9529 sys_catalog.cc:458] T 00000000000000000000000000000000 P 62e54dd7499440acaa1b1c76403da552 [sys.catalog]: This master's current role is: FOLLOWER
08:07:49.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:49.478109 9524 sys_catalog.cc:565] T 00000000000000000000000000000000 P 62e54dd7499440acaa1b1c76403da552 [sys.catalog]: configured and running, proceeding with master startup.
08:07:49.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 23 08:07:49 dist-test-slave-wxqn krb5kdc[8555](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761206868, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.87.193@KRBTEST.COM for kudu/127.8.87.254@KRBTEST.COM
08:07:49.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:49.588104 9029 heartbeater.cc:344] Connected to a master server at 127.8.87.254:43115
08:07:49.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:49.595623 9489 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "b89b7d622ef34a1cb032858329be126a" instance_seqno: 1761206859197829) as {username='kudu', principal='kudu/127.8.87.193@KRBTEST.COM'} at 127.8.87.193:43383; Asking this server to re-register.
08:07:49.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:49.597309 9029 heartbeater.cc:461] Registering TS with master...
08:07:49.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:49.601564 9489 ts_manager.cc:194] Registered new tserver with Master: b89b7d622ef34a1cb032858329be126a (127.8.87.193:36301)
08:07:49.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:49.628644 9543 catalog_manager.cc:1269] Loaded cluster ID: 8f877e344eb84cea8a008e14c9bc53c8
08:07:49.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:49.628966 9543 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 62e54dd7499440acaa1b1c76403da552: loading cluster ID for follower catalog manager: success
08:07:49.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 23 08:07:49 dist-test-slave-wxqn krb5kdc[8555](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761206869, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.87.254@KRBTEST.COM for kudu/127.8.87.254@KRBTEST.COM
08:07:49.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:49.636967 9543 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 62e54dd7499440acaa1b1c76403da552: acquiring CA information for follower catalog manager: success
08:07:49.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:49.642385 9543 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 62e54dd7499440acaa1b1c76403da552: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
08:07:49.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:49.656637 9421 master_runner.cc:186] Error getting master registration for 127.8.87.253:46439: Network error: Client connection negotiation failed: client connection to 127.8.87.253:46439: connect: Connection refused (error 111),
08:07:49.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:49.683393 9421 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
08:07:49.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:49.823748 9323 heartbeater.cc:344] Connected to a master server at 127.8.87.254:43115
08:07:49.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:49.826208 9489 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "2c485d5a4b9f4ba6926df4b0102eed27" instance_seqno: 1761206864566991) as {username='kudu', principal='kudu/127.8.87.195@KRBTEST.COM'} at 127.8.87.195:33643; Asking this server to re-register.
08:07:49.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:49.827665 9323 heartbeater.cc:461] Registering TS with master...
08:07:49.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:49.830394 9489 ts_manager.cc:194] Registered new tserver with Master: 2c485d5a4b9f4ba6926df4b0102eed27 (127.8.87.195:36087)
08:07:49.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:49.849155 9527 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:07:49.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:49.849805 9527 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:07:49.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:49.850579 9527 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:07:49.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:49.886313 9527 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
08:07:49.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:49.886653 9527 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
08:07:49.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:49.887110 9527 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:07:49.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:49.887429 9527 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
08:07:49.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:49.887953 9527 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
08:07:49.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:49.888295 9527 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
08:07:49.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:49.888549 9527 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:40533/jwks.json
08:07:49.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:49.928453 9527 master_runner.cc:386] Master server non-default flags:
08:07:49.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:07:49.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.87.212:46335
08:07:49.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:07:49.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:07:49.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-1/data
08:07:49.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-1/wal
08:07:49.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
08:07:49.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:07:49.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.87.254:43115,127.8.87.253:46439,127.8.87.252:38507
08:07:49.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/krb5kdc/kudu.keytab
08:07:49.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.8.87.253
08:07:49.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:07:49.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:07:49.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:07:49.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.87.253:46439
08:07:49.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:07:49.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:07:49.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:07:49.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40533/jwks.json
08:07:49.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:07:49.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
08:07:49.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
08:07:49.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:07:49.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:07:49.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-1/data/info.pb
08:07:49.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.87.253
08:07:49.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44589
08:07:49.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:07:49.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:07:49.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:07:49.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:07:49.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:07:49.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-1/logs
08:07:49.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:07:49.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:07:49.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:07:49.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:07:49.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:07:49.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 18b1930e830609248ee2e26d6216d073ff40ac5d
08:07:49.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:07:49.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 23 Oct 2025 07:43:21 UTC on 5fd53c4cbb9d
08:07:49.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8677
08:07:49.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:07:49.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:49.930136 9527 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:07:49.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:49.931685 9527 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:07:49.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:49.950203 9557 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:07:50.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:50.146637 9563 raft_consensus.cc:493] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:07:50.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:50.147735 9563 raft_consensus.cc:515] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "62e54dd7499440acaa1b1c76403da552" member_type: VOTER last_known_addr { host: "127.8.87.254" port: 43115 } } peers { permanent_uuid: "4283b8cd94454e13bb77279c6afdb8f2" member_type: VOTER last_known_addr { host: "127.8.87.253" port: 46439 } } peers { permanent_uuid: "b0ae399fb47a454996698251cea1c12d" member_type: VOTER last_known_addr { host: "127.8.87.252" port: 38507 } }
08:07:50.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:50.154289 9563 leader_election.cc:290] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 62e54dd7499440acaa1b1c76403da552 (127.8.87.254:43115), 4283b8cd94454e13bb77279c6afdb8f2 (127.8.87.253:46439)
08:07:50.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:50.176070 9370 leader_election.cc:336] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 4283b8cd94454e13bb77279c6afdb8f2 (127.8.87.253:46439): Network error: Client connection negotiation failed: client connection to 127.8.87.253:46439: connect: Connection refused (error 111)
08:07:50.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:50.183921 9499 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "b0ae399fb47a454996698251cea1c12d" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "62e54dd7499440acaa1b1c76403da552" is_pre_election: true
08:07:50.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:50.185103 9499 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 62e54dd7499440acaa1b1c76403da552 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate b0ae399fb47a454996698251cea1c12d in term 1.
08:07:50.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:50.187012 9368 leader_election.cc:304] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 62e54dd7499440acaa1b1c76403da552, b0ae399fb47a454996698251cea1c12d; no voters: 4283b8cd94454e13bb77279c6afdb8f2
08:07:50.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:50.188181 9563 raft_consensus.cc:2804] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d [term 1 FOLLOWER]: Leader pre-election won for term 2
08:07:50.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:50.188639 9563 raft_consensus.cc:493] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d [term 1 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
08:07:50.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:50.189074 9563 raft_consensus.cc:3060] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d [term 1 FOLLOWER]: Advancing to term 2
08:07:50.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:50.208885 9563 raft_consensus.cc:515] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "62e54dd7499440acaa1b1c76403da552" member_type: VOTER last_known_addr { host: "127.8.87.254" port: 43115 } } peers { permanent_uuid: "4283b8cd94454e13bb77279c6afdb8f2" member_type: VOTER last_known_addr { host: "127.8.87.253" port: 46439 } } peers { permanent_uuid: "b0ae399fb47a454996698251cea1c12d" member_type: VOTER last_known_addr { host: "127.8.87.252" port: 38507 } }
08:07:50.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:50.212633 9499 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "b0ae399fb47a454996698251cea1c12d" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "62e54dd7499440acaa1b1c76403da552"
08:07:50.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:50.213372 9499 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 62e54dd7499440acaa1b1c76403da552 [term 1 FOLLOWER]: Advancing to term 2
08:07:50.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:50.218274 9563 leader_election.cc:290] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d [CANDIDATE]: Term 2 election: Requested vote from peers 62e54dd7499440acaa1b1c76403da552 (127.8.87.254:43115), 4283b8cd94454e13bb77279c6afdb8f2 (127.8.87.253:46439)
08:07:50.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:50.219239 9370 leader_election.cc:336] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 4283b8cd94454e13bb77279c6afdb8f2 (127.8.87.253:46439): Network error: Client connection negotiation failed: client connection to 127.8.87.253:46439: connect: Connection refused (error 111)
08:07:50.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:50.226303 9499 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 62e54dd7499440acaa1b1c76403da552 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate b0ae399fb47a454996698251cea1c12d in term 2.
08:07:50.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:50.244859 9368 leader_election.cc:304] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 62e54dd7499440acaa1b1c76403da552, b0ae399fb47a454996698251cea1c12d; no voters: 4283b8cd94454e13bb77279c6afdb8f2
08:07:50.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:50.245939 9563 raft_consensus.cc:2804] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d [term 2 FOLLOWER]: Leader election won for term 2
08:07:50.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:50.267339 9563 raft_consensus.cc:697] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d [term 2 LEADER]: Becoming Leader. State: Replica: b0ae399fb47a454996698251cea1c12d, State: Running, Role: LEADER
08:07:50.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:50.268909 9563 consensus_queue.cc:237] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "62e54dd7499440acaa1b1c76403da552" member_type: VOTER last_known_addr { host: "127.8.87.254" port: 43115 } } peers { permanent_uuid: "4283b8cd94454e13bb77279c6afdb8f2" member_type: VOTER last_known_addr { host: "127.8.87.253" port: 46439 } } peers { permanent_uuid: "b0ae399fb47a454996698251cea1c12d" member_type: VOTER last_known_addr { host: "127.8.87.252" port: 38507 } }
08:07:50.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:50.276515 9176 heartbeater.cc:344] Connected to a master server at 127.8.87.254:43115
08:07:50.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:50.279671 9489 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "0b8b4a4be8314be4b54168e19654fe19" instance_seqno: 1761206862002646) as {username='kudu', principal='kudu/127.8.87.194@KRBTEST.COM'} at 127.8.87.194:46819; Asking this server to re-register.
08:07:50.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:50.286240 9176 heartbeater.cc:461] Registering TS with master...
08:07:50.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:50.289748 9489 ts_manager.cc:194] Registered new tserver with Master: 0b8b4a4be8314be4b54168e19654fe19 (127.8.87.194:43889)
08:07:50.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:50.295464 9568 sys_catalog.cc:455] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d [sys.catalog]: SysCatalogTable state changed. Reason: New leader b0ae399fb47a454996698251cea1c12d. Latest consensus state: current_term: 2 leader_uuid: "b0ae399fb47a454996698251cea1c12d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "62e54dd7499440acaa1b1c76403da552" member_type: VOTER last_known_addr { host: "127.8.87.254" port: 43115 } } peers { permanent_uuid: "4283b8cd94454e13bb77279c6afdb8f2" member_type: VOTER last_known_addr { host: "127.8.87.253" port: 46439 } } peers { permanent_uuid: "b0ae399fb47a454996698251cea1c12d" member_type: VOTER last_known_addr { host: "127.8.87.252" port: 38507 } } }
08:07:50.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:50.296694 9568 sys_catalog.cc:458] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d [sys.catalog]: This master's current role is: LEADER
08:07:50.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:50.324152 9571 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
08:07:50.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:50.346804 9571 catalog_manager.cc:1494] Initializing Kudu cluster ID...
08:07:50.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:50.361476 9571 catalog_manager.cc:1269] Loaded cluster ID: 8f877e344eb84cea8a008e14c9bc53c8
08:07:50.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:50.361840 9571 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
08:07:50.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:50.376387 9571 catalog_manager.cc:1514] Loading token signing keys...
08:07:50.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:50.385988 9571 catalog_manager.cc:6033] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d: Loaded TSK: 0
08:07:50.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:50.387727 9571 catalog_manager.cc:1524] Initializing in-progress tserver states...
08:07:50.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:50.664273 9370 consensus_peers.cc:597] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d -> Peer 4283b8cd94454e13bb77279c6afdb8f2 (127.8.87.253:46439): Couldn't send request to peer 4283b8cd94454e13bb77279c6afdb8f2. Status: Network error: Client connection negotiation failed: client connection to 127.8.87.253:46439: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
08:07:50.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:50.725481 9499 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 62e54dd7499440acaa1b1c76403da552 [term 2 FOLLOWER]: Refusing update from remote peer b0ae399fb47a454996698251cea1c12d: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 2 index: 5. (index mismatch)
08:07:50.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:50.727651 9568 consensus_queue.cc:1048] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d [LEADER]: Connected to new peer: Peer: permanent_uuid: "62e54dd7499440acaa1b1c76403da552" member_type: VOTER last_known_addr { host: "127.8.87.254" port: 43115 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.001s
08:07:50.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:50.774987 9575 sys_catalog.cc:455] T 00000000000000000000000000000000 P 62e54dd7499440acaa1b1c76403da552 [sys.catalog]: SysCatalogTable state changed. Reason: New leader b0ae399fb47a454996698251cea1c12d. Latest consensus state: current_term: 2 leader_uuid: "b0ae399fb47a454996698251cea1c12d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "62e54dd7499440acaa1b1c76403da552" member_type: VOTER last_known_addr { host: "127.8.87.254" port: 43115 } } peers { permanent_uuid: "4283b8cd94454e13bb77279c6afdb8f2" member_type: VOTER last_known_addr { host: "127.8.87.253" port: 46439 } } peers { permanent_uuid: "b0ae399fb47a454996698251cea1c12d" member_type: VOTER last_known_addr { host: "127.8.87.252" port: 38507 } } }
08:07:50.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:50.776448 9575 sys_catalog.cc:458] T 00000000000000000000000000000000 P 62e54dd7499440acaa1b1c76403da552 [sys.catalog]: This master's current role is: FOLLOWER
08:07:50.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:50.790503 9568 sys_catalog.cc:455] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "b0ae399fb47a454996698251cea1c12d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "62e54dd7499440acaa1b1c76403da552" member_type: VOTER last_known_addr { host: "127.8.87.254" port: 43115 } } peers { permanent_uuid: "4283b8cd94454e13bb77279c6afdb8f2" member_type: VOTER last_known_addr { host: "127.8.87.253" port: 46439 } } peers { permanent_uuid: "b0ae399fb47a454996698251cea1c12d" member_type: VOTER last_known_addr { host: "127.8.87.252" port: 38507 } } }
08:07:50.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:50.791239 9568 sys_catalog.cc:458] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d [sys.catalog]: This master's current role is: LEADER
08:07:50.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:50.793632 9575 sys_catalog.cc:455] T 00000000000000000000000000000000 P 62e54dd7499440acaa1b1c76403da552 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "b0ae399fb47a454996698251cea1c12d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "62e54dd7499440acaa1b1c76403da552" member_type: VOTER last_known_addr { host: "127.8.87.254" port: 43115 } } peers { permanent_uuid: "4283b8cd94454e13bb77279c6afdb8f2" member_type: VOTER last_known_addr { host: "127.8.87.253" port: 46439 } } peers { permanent_uuid: "b0ae399fb47a454996698251cea1c12d" member_type: VOTER last_known_addr { host: "127.8.87.252" port: 38507 } } }
08:07:50.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:50.794528 9575 sys_catalog.cc:458] T 00000000000000000000000000000000 P 62e54dd7499440acaa1b1c76403da552 [sys.catalog]: This master's current role is: FOLLOWER
08:07:51.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:51.505736 9025 heartbeater.cc:499] Master 127.8.87.252:38507 was elected leader, sending a full tablet report...
08:07:51.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 23 08:07:51 dist-test-slave-wxqn krb5kdc[8555](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761206871, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.87.194@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:07:51.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:51.735602 9056 init.cc:295] Successfully reacquired a new kerberos TGT
08:07:51.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:51.825692 9322 heartbeater.cc:499] Master 127.8.87.252:38507 was elected leader, sending a full tablet report...
08:07:52.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:52.189379 9337 master_runner.cc:186] Error getting master registration for 127.8.87.253:46439: Network error: Client connection negotiation failed: client connection to 127.8.87.253:46439: connect: Connection refused (error 111),
08:07:52.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:52.190001 9337 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
08:07:52.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:51.351787 9556 debug-util.cc:398] Leaking SignalData structure 0x7b0800027080 after lost signal to thread 9527
08:07:52.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:52.225023 9527 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 2.267s user 0.002s sys 0.000s
08:07:52.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:49.956832 9558 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:07:52.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:52.229944 9559 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2266 milliseconds
08:07:52.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:52.229249 9527 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 2.271s user 0.002s sys 0.005s
08:07:52.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:52.231606 9527 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
08:07:52.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:52.231653 9561 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:07:52.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:52.234769 9527 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:07:52.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:52.237738 9527 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:07:52.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:52.239272 9527 hybrid_clock.cc:648] HybridClock initialized: now 1761206872239238 us; error 46 us; skew 500 ppm
08:07:52.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:52.240074 9527 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:07:52.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 23 08:07:52 dist-test-slave-wxqn krb5kdc[8555](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761206872, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.87.253@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:07:52.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:52.246703 9527 init.cc:377] Logged in from keytab as kudu/127.8.87.253@KRBTEST.COM (short username kudu)
08:07:52.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:52.261364 9527 webserver.cc:492] Webserver started at http://127.8.87.253:44589/ using document root <none> and password file <none>
08:07:52.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:52.262436 9527 fs_manager.cc:362] Metadata directory not provided
08:07:52.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:52.262710 9527 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:07:52.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:52.271202 9527 fs_manager.cc:714] Time spent opening directory manager: real 0.006s user 0.001s sys 0.004s
08:07:52.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:52.276199 9593 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:07:52.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:52.277498 9527 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
08:07:52.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:52.277797 9527 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-1/data,/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-1/wal
08:07:52.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4283b8cd94454e13bb77279c6afdb8f2"
08:07:52.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-23 08:07:31 on dist-test-slave-wxqn"
08:07:52.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:52.279767 9527 fs_report.cc:389] FS layout report
08:07:52.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:07:52.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-1/wal
08:07:52.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-1/wal
08:07:52.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-1/data/data
08:07:52.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:07:52.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:07:52.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:07:52.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:07:52.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:07:52.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:07:52.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:07:52.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:07:52.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:07:52.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:07:52.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:07:52.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:52.313514 9175 heartbeater.cc:499] Master 127.8.87.252:38507 was elected leader, sending a full tablet report...
08:07:52.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:52.352017 9527 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:07:52.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:52.362661 9527 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:07:52.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:52.363164 9527 kserver.cc:163] Server-wide thread pool size limit: 3276
08:07:52.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:52.483785 9527 rpc_server.cc:307] RPC server started. Bound to: 127.8.87.253:46439
08:07:52.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:52.483843 9646 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.8.87.253:46439 every 8 connection(s)
08:07:52.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:52.486847 9527 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-1/data/info.pb
08:07:52.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:52.492784 8543 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskcrgkbR/build/tsan/bin/kudu as pid 9527
08:07:52.493 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:07:52.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:52.500511 9647 sys_catalog.cc:263] Verifying existing consensus state
08:07:52.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:52.505808 9647 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4283b8cd94454e13bb77279c6afdb8f2: Bootstrap starting.
08:07:52.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:52.556164 7891 debug-util.cc:398] Leaking SignalData structure 0x7b0800090c40 after lost signal to thread 7828
08:07:52.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:52.556916 7891 debug-util.cc:398] Leaking SignalData structure 0x7b080009c420 after lost signal to thread 7894
08:07:52.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:52.561892 9647 log.cc:826] T 00000000000000000000000000000000 P 4283b8cd94454e13bb77279c6afdb8f2: Log is configured to *not* fsync() on all Append() calls
08:07:52.564 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.8.87.252:38507(127.8.87.252:38507)] connecting to peer
08:07:52.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:52.588018 9647 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4283b8cd94454e13bb77279c6afdb8f2: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
08:07:52.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:52.589558 9647 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4283b8cd94454e13bb77279c6afdb8f2: Bootstrap complete.
08:07:52.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 23 08:07:52 dist-test-slave-wxqn krb5kdc[8555](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761206868, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.87.193@KRBTEST.COM for kudu/127.8.87.253@KRBTEST.COM
08:07:52.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 23 08:07:52 dist-test-slave-wxqn krb5kdc[8555](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761206866, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.87.252@KRBTEST.COM for kudu/127.8.87.253@KRBTEST.COM
08:07:52.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:52.694636 9024 heartbeater.cc:344] Connected to a master server at 127.8.87.253:46439
08:07:52.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:52.703038 9647 raft_consensus.cc:359] T 00000000000000000000000000000000 P 4283b8cd94454e13bb77279c6afdb8f2 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "62e54dd7499440acaa1b1c76403da552" member_type: VOTER last_known_addr { host: "127.8.87.254" port: 43115 } } peers { permanent_uuid: "4283b8cd94454e13bb77279c6afdb8f2" member_type: VOTER last_known_addr { host: "127.8.87.253" port: 46439 } } peers { permanent_uuid: "b0ae399fb47a454996698251cea1c12d" member_type: VOTER last_known_addr { host: "127.8.87.252" port: 38507 } }
08:07:52.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:52.712416 9647 raft_consensus.cc:740] T 00000000000000000000000000000000 P 4283b8cd94454e13bb77279c6afdb8f2 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4283b8cd94454e13bb77279c6afdb8f2, State: Initialized, Role: FOLLOWER
08:07:52.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:52.714180 9647 consensus_queue.cc:260] T 00000000000000000000000000000000 P 4283b8cd94454e13bb77279c6afdb8f2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "62e54dd7499440acaa1b1c76403da552" member_type: VOTER last_known_addr { host: "127.8.87.254" port: 43115 } } peers { permanent_uuid: "4283b8cd94454e13bb77279c6afdb8f2" member_type: VOTER last_known_addr { host: "127.8.87.253" port: 46439 } } peers { permanent_uuid: "b0ae399fb47a454996698251cea1c12d" member_type: VOTER last_known_addr { host: "127.8.87.252" port: 38507 } }
08:07:52.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:52.765166 9647 sys_catalog.cc:565] T 00000000000000000000000000000000 P 4283b8cd94454e13bb77279c6afdb8f2 [sys.catalog]: configured and running, proceeding with master startup.
08:07:52.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:52.765981 9621 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 4283b8cd94454e13bb77279c6afdb8f2 [term 1 FOLLOWER]: Advancing to term 2
08:07:52.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:52.762472 9655 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4283b8cd94454e13bb77279c6afdb8f2 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "62e54dd7499440acaa1b1c76403da552" member_type: VOTER last_known_addr { host: "127.8.87.254" port: 43115 } } peers { permanent_uuid: "4283b8cd94454e13bb77279c6afdb8f2" member_type: VOTER last_known_addr { host: "127.8.87.253" port: 46439 } } peers { permanent_uuid: "b0ae399fb47a454996698251cea1c12d" member_type: VOTER last_known_addr { host: "127.8.87.252" port: 38507 } } }
08:07:52.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:52.777143 9655 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4283b8cd94454e13bb77279c6afdb8f2 [sys.catalog]: This master's current role is: FOLLOWER
08:07:52.784 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.8.87.254:43115(127.8.87.254:43115)] connecting to peer
08:07:52.808 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.8.87.253:46439(127.8.87.253:46439)] connecting to peer
08:07:52.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:52.851943 9655 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4283b8cd94454e13bb77279c6afdb8f2 [sys.catalog]: SysCatalogTable state changed. Reason: New leader b0ae399fb47a454996698251cea1c12d. Latest consensus state: current_term: 2 leader_uuid: "b0ae399fb47a454996698251cea1c12d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "62e54dd7499440acaa1b1c76403da552" member_type: VOTER last_known_addr { host: "127.8.87.254" port: 43115 } } peers { permanent_uuid: "4283b8cd94454e13bb77279c6afdb8f2" member_type: VOTER last_known_addr { host: "127.8.87.253" port: 46439 } } peers { permanent_uuid: "b0ae399fb47a454996698251cea1c12d" member_type: VOTER last_known_addr { host: "127.8.87.252" port: 38507 } } }
08:07:52.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:52.852667 9655 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4283b8cd94454e13bb77279c6afdb8f2 [sys.catalog]: This master's current role is: FOLLOWER
08:07:52.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:52.851159 9667 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4283b8cd94454e13bb77279c6afdb8f2 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "b0ae399fb47a454996698251cea1c12d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "62e54dd7499440acaa1b1c76403da552" member_type: VOTER last_known_addr { host: "127.8.87.254" port: 43115 } } peers { permanent_uuid: "4283b8cd94454e13bb77279c6afdb8f2" member_type: VOTER last_known_addr { host: "127.8.87.253" port: 46439 } } peers { permanent_uuid: "b0ae399fb47a454996698251cea1c12d" member_type: VOTER last_known_addr { host: "127.8.87.252" port: 38507 } } }
08:07:52.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:52.853789 9667 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4283b8cd94454e13bb77279c6afdb8f2 [sys.catalog]: This master's current role is: FOLLOWER
08:07:52.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:52.870337 9672 catalog_manager.cc:805] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
08:07:52.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:52.905138 9321 heartbeater.cc:344] Connected to a master server at 127.8.87.253:46439
08:07:52.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:52.905853 9321 heartbeater.cc:499] Master 127.8.87.253:46439 was elected leader, sending a full tablet report...
08:07:52.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:52.912873 9611 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "2c485d5a4b9f4ba6926df4b0102eed27" instance_seqno: 1761206864566991) as {username='kudu', principal='kudu/127.8.87.195@KRBTEST.COM'} at 127.8.87.195:35471; Asking this server to re-register.
08:07:52.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:52.914459 9321 heartbeater.cc:461] Registering TS with master...
08:07:52.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:52.918713 9611 ts_manager.cc:194] Registered new tserver with Master: 2c485d5a4b9f4ba6926df4b0102eed27 (127.8.87.195:36087)
08:07:52.922 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.8.87.254:43115(127.8.87.254:43115)] Successfully connected to peer
08:07:52.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 23 08:07:52 dist-test-slave-wxqn krb5kdc[8555](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761206872, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.87.253@KRBTEST.COM for kudu/127.8.87.254@KRBTEST.COM
08:07:52.952 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.8.87.252:38507(127.8.87.252:38507)] Successfully connected to peer
08:07:52.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 23 08:07:52 dist-test-slave-wxqn krb5kdc[8555](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761206872, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.87.253@KRBTEST.COM for kudu/127.8.87.253@KRBTEST.COM
08:07:53.030 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.8.87.253:46439(127.8.87.253:46439)] Successfully connected to peer
08:07:53.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 23 08:07:53 dist-test-slave-wxqn krb5kdc[8555](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761206872, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.87.253@KRBTEST.COM for kudu/127.8.87.252@KRBTEST.COM
08:07:53.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:53.080397 9563 sys_catalog.cc:455] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "b0ae399fb47a454996698251cea1c12d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "62e54dd7499440acaa1b1c76403da552" member_type: VOTER last_known_addr { host: "127.8.87.254" port: 43115 } } peers { permanent_uuid: "4283b8cd94454e13bb77279c6afdb8f2" member_type: VOTER last_known_addr { host: "127.8.87.253" port: 46439 } } peers { permanent_uuid: "b0ae399fb47a454996698251cea1c12d" member_type: VOTER last_known_addr { host: "127.8.87.252" port: 38507 } } }
08:07:53.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:53.083079 9563 sys_catalog.cc:458] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d [sys.catalog]: This master's current role is: LEADER
08:07:53.091 [WARN - kudu-nio-1] (SecurityContext.java:206) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
08:07:53.109 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.8.87.254
08:07:53.117 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.8.87.252
08:07:53.258 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.8.87.253
08:07:53.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 23 08:07:53 dist-test-slave-wxqn krb5kdc[8555](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761206871, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.87.194@KRBTEST.COM for kudu/127.8.87.253@KRBTEST.COM
08:07:53.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:53.430302 9174 heartbeater.cc:344] Connected to a master server at 127.8.87.253:46439
08:07:53.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:53.433220 9611 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "0b8b4a4be8314be4b54168e19654fe19" instance_seqno: 1761206862002646) as {username='kudu', principal='kudu/127.8.87.194@KRBTEST.COM'} at 127.8.87.194:49249; Asking this server to re-register.
08:07:53.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:53.434888 9174 heartbeater.cc:461] Registering TS with master...
08:07:53.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:53.437848 9611 ts_manager.cc:194] Registered new tserver with Master: 0b8b4a4be8314be4b54168e19654fe19 (127.8.87.194:43889)
08:07:53.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 23 08:07:53 dist-test-slave-wxqn krb5kdc[8555](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761206864, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.8.87.254@KRBTEST.COM
08:07:53.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 23 08:07:53 dist-test-slave-wxqn krb5kdc[8555](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761206864, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.8.87.252@KRBTEST.COM
08:07:53.670 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x6f83972b, L:/127.0.0.1:36414 - R:/127.8.87.254:43115] using SASL/GSSAPI
08:07:53.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:53.701567 9611 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "b89b7d622ef34a1cb032858329be126a" instance_seqno: 1761206859197829) as {username='kudu', principal='kudu/127.8.87.193@KRBTEST.COM'} at 127.8.87.193:47835; Asking this server to re-register.
08:07:53.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 23 08:07:53 dist-test-slave-wxqn krb5kdc[8555](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761206864, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.8.87.253@KRBTEST.COM
08:07:53.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:53.703490 9024 heartbeater.cc:461] Registering TS with master...
08:07:53.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:53.707262 9611 ts_manager.cc:194] Registered new tserver with Master: b89b7d622ef34a1cb032858329be126a (127.8.87.193:36301)
08:07:53.744 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x12f6d7d0, L:/127.0.0.1:34966 - R:/127.8.87.252:38507] using SASL/GSSAPI
08:07:53.751 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xd2c67308, L:/127.0.0.1:43232 - R:/127.8.87.253:46439] using SASL/GSSAPI
08:07:53.808 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
08:07:53.810 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=ListTabletServers, tablet=null, attempt=1, TimeoutTracker(timeout=30000, elapsed=1311), Traces: [0ms] refreshing cache from master, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.87.252:38507, [279ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.87.254:43115, [304ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.87.253:46439, [1184ms] Sub RPC ConnectToMaster: received response from server master-127.8.87.254:43115: OK, [1251ms] Sub RPC ConnectToMaster: received response from server master-127.8.87.253:46439: OK, [1262ms] Sub RPC ConnectToMaster: received response from server master-127.8.87.252:38507: OK, deferred=null) after lookup
08:07:53.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:53.897980 9672 catalog_manager.cc:1269] Loaded cluster ID: 8f877e344eb84cea8a008e14c9bc53c8
08:07:53.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:53.898440 9672 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 4283b8cd94454e13bb77279c6afdb8f2: loading cluster ID for follower catalog manager: success
08:07:53.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:53.906355 9672 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 4283b8cd94454e13bb77279c6afdb8f2: acquiring CA information for follower catalog manager: success
08:07:53.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:53.912111 9672 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 4283b8cd94454e13bb77279c6afdb8f2: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
08:07:54.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 23 08:07:54 dist-test-slave-wxqn krb5kdc[8555](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761206874, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.87.195@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:07:54.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:54.128823 9204 init.cc:295] Successfully reacquired a new kerberos TGT
08:07:54.849 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
08:07:54.850 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
08:07:54.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 23 08:07:54 dist-test-slave-wxqn krb5kdc[8555](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761206874, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:07:54.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
08:07:54.880 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:07:54.880 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/krb5kdc/krb5cc
08:07:54.881 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
08:07:54.882 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.8.87.252:38507
08:07:54.882 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
08:07:54.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:54.882534 8543 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskcrgkbR/build/tsan/bin/kudu with pid 9337
08:07:54.923 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:07:54.924 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.8.87.254:43115
08:07:54.924 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
08:07:54.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:54.924747 8543 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskcrgkbR/build/tsan/bin/kudu with pid 9421
08:07:54.931 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.8.87.252:38507(127.8.87.252:38507)] handling channelInactive
08:07:54.932 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.87.252:38507(127.8.87.252:38507)] cleaning up while in state READY due to: connection closed
08:07:54.961 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.8.87.254:43115(127.8.87.254:43115)] handling channelInactive
08:07:54.961 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.87.254:43115(127.8.87.254:43115)] cleaning up while in state READY due to: connection closed
08:07:54.962 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:07:54.962 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.8.87.253:46439
08:07:54.962 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
08:07:54.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:54.963269 8543 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskcrgkbR/build/tsan/bin/kudu with pid 9527
08:07:54.994 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:07:54.995 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.8.87.252:38507
08:07:54.995 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
08:07:54.995 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.8.87.253:46439(127.8.87.253:46439)] handling channelInactive
08:07:54.996 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.87.253:46439(127.8.87.253:46439)] cleaning up while in state READY due to: connection closed
08:07:54.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:54.997081 8543 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskcrgkbR/build/tsan/bin/kudu
08:07:54.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskcrgkbR/build/tsan/bin/kudu
08:07:54.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-2/wal
08:07:54.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-2/data
08:07:54.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:07:54.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:07:54.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:07:54.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:07:54.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:07:54.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:07:54.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-2/logs
08:07:54.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-2/data/info.pb
08:07:54.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:07:54.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:07:54.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:07:54.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:07:54.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:07:54.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:07:54.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:07:54.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:07:54.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:07:54.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:07:54.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:07:54.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:07:54.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.87.252:38507
08:07:54.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.87.252
08:07:54.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=40629
08:07:54.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/krb5kdc/kudu.keytab
08:07:54.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.8.87.252
08:07:54.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:07:54.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
08:07:54.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
08:07:54.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
08:07:54.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.87.212:46335
08:07:54.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:07:54.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:07:54.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:07:54.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:07:54.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.87.254:43115,127.8.87.253:46439,127.8.87.252:38507
08:07:54.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:07:54.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40533/jwks.json
08:07:54.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
08:07:54.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
08:07:55.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:55.313652 9691 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:07:55.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:55.314208 9691 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:07:55.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:55.314824 9691 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:07:55.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:55.351461 9691 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
08:07:55.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:55.351814 9691 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
08:07:55.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:55.352259 9691 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:07:55.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:55.352622 9691 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
08:07:55.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:55.352936 9691 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
08:07:55.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:55.353243 9691 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
08:07:55.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:55.353472 9691 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:40533/jwks.json
08:07:55.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:55.408035 9691 master_runner.cc:386] Master server non-default flags:
08:07:55.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:07:55.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.87.212:46335
08:07:55.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:07:55.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:07:55.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-2/data
08:07:55.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-2/wal
08:07:55.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
08:07:55.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:07:55.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.87.254:43115,127.8.87.253:46439,127.8.87.252:38507
08:07:55.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/krb5kdc/kudu.keytab
08:07:55.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.8.87.252
08:07:55.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:07:55.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:07:55.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:07:55.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.87.252:38507
08:07:55.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:07:55.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:07:55.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:07:55.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40533/jwks.json
08:07:55.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:07:55.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
08:07:55.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
08:07:55.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:07:55.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:07:55.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-2/data/info.pb
08:07:55.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.87.252
08:07:55.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=40629
08:07:55.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:07:55.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:07:55.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:07:55.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:07:55.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:07:55.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-2/logs
08:07:55.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:07:55.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:07:55.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:07:55.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:07:55.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:07:55.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 18b1930e830609248ee2e26d6216d073ff40ac5d
08:07:55.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:07:55.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 23 Oct 2025 07:43:21 UTC on 5fd53c4cbb9d
08:07:55.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8677
08:07:55.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:07:55.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:55.409785 9691 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:07:55.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:55.411146 9691 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:07:55.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:55.424908 9698 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:07:56.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:55.427742 9699 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:07:56.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:55.431282 9701 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:07:56.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:56.725911 9700 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1296 milliseconds
08:07:56.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:56.726019 9691 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
08:07:56.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:56.727178 9691 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:07:56.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:56.729676 9691 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:07:56.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:56.731081 9691 hybrid_clock.cc:648] HybridClock initialized: now 1761206876730971 us; error 108 us; skew 500 ppm
08:07:56.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:56.731871 9691 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:07:56.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 23 08:07:56 dist-test-slave-wxqn krb5kdc[8555](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761206876, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.87.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:07:56.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:56.738131 9691 init.cc:377] Logged in from keytab as kudu/127.8.87.252@KRBTEST.COM (short username kudu)
08:07:56.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:56.743775 9691 webserver.cc:492] Webserver started at http://127.8.87.252:40629/ using document root <none> and password file <none>
08:07:56.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:56.744788 9691 fs_manager.cc:362] Metadata directory not provided
08:07:56.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:56.745013 9691 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:07:56.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:56.753475 9691 fs_manager.cc:714] Time spent opening directory manager: real 0.006s user 0.003s sys 0.005s
08:07:56.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:56.757997 9718 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:07:56.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:56.759090 9691 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.000s sys 0.004s
08:07:56.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:56.759429 9691 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-2/data,/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-2/wal
08:07:56.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b0ae399fb47a454996698251cea1c12d"
08:07:56.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-23 08:07:34 on dist-test-slave-wxqn"
08:07:56.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:56.761529 9691 fs_report.cc:389] FS layout report
08:07:56.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:07:56.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-2/wal
08:07:56.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-2/wal
08:07:56.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-2/data/data
08:07:56.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:07:56.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:07:56.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:07:56.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:07:56.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:07:56.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:07:56.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:07:56.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:07:56.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:07:56.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:07:56.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:07:56.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:56.815861 9691 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:07:56.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:56.821758 9691 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:07:56.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:56.822175 9691 kserver.cc:163] Server-wide thread pool size limit: 3276
08:07:56.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:56.893527 9691 rpc_server.cc:307] RPC server started. Bound to: 127.8.87.252:38507
08:07:56.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:56.893579 9770 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.8.87.252:38507 every 8 connection(s)
08:07:56.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:56.896345 9691 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-2/data/info.pb
08:07:56.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:56.898540 8543 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskcrgkbR/build/tsan/bin/kudu as pid 9691
08:07:56.899 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:07:56.899 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.8.87.254:43115
08:07:56.900 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
08:07:56.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:56.901257 8543 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskcrgkbR/build/tsan/bin/kudu
08:07:56.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskcrgkbR/build/tsan/bin/kudu
08:07:56.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-0/wal
08:07:56.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-0/data
08:07:56.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:07:56.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:07:56.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:07:56.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:07:56.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:07:56.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:07:56.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-0/logs
08:07:56.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-0/data/info.pb
08:07:56.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:07:56.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:07:56.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:07:56.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:07:56.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:07:56.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:07:56.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:07:56.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:07:56.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:07:56.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:07:56.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:07:56.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:07:56.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.87.254:43115
08:07:56.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.87.254
08:07:56.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33423
08:07:56.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/krb5kdc/kudu.keytab
08:07:56.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.8.87.254
08:07:56.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:07:56.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
08:07:56.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
08:07:56.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
08:07:56.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.87.212:46335
08:07:56.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:07:56.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:07:56.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:07:56.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:07:56.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.87.254:43115,127.8.87.253:46439,127.8.87.252:38507
08:07:56.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:07:56.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40533/jwks.json
08:07:56.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
08:07:56.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
08:07:56.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:56.908772 9771 sys_catalog.cc:263] Verifying existing consensus state
08:07:56.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:56.914474 9771 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d: Bootstrap starting.
08:07:56.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:56.959784 9771 log.cc:826] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d: Log is configured to *not* fsync() on all Append() calls
08:07:56.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:56.976606 9771 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
08:07:56.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:56.978276 9771 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d: Bootstrap complete.
08:07:57.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:57.009994 9771 raft_consensus.cc:359] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "62e54dd7499440acaa1b1c76403da552" member_type: VOTER last_known_addr { host: "127.8.87.254" port: 43115 } } peers { permanent_uuid: "4283b8cd94454e13bb77279c6afdb8f2" member_type: VOTER last_known_addr { host: "127.8.87.253" port: 46439 } } peers { permanent_uuid: "b0ae399fb47a454996698251cea1c12d" member_type: VOTER last_known_addr { host: "127.8.87.252" port: 38507 } }
08:07:57.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:57.012534 9771 raft_consensus.cc:740] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: b0ae399fb47a454996698251cea1c12d, State: Initialized, Role: FOLLOWER
08:07:57.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:57.013599 9771 consensus_queue.cc:260] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 2.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "62e54dd7499440acaa1b1c76403da552" member_type: VOTER last_known_addr { host: "127.8.87.254" port: 43115 } } peers { permanent_uuid: "4283b8cd94454e13bb77279c6afdb8f2" member_type: VOTER last_known_addr { host: "127.8.87.253" port: 46439 } } peers { permanent_uuid: "b0ae399fb47a454996698251cea1c12d" member_type: VOTER last_known_addr { host: "127.8.87.252" port: 38507 } }
08:07:57.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:57.016238 9776 sys_catalog.cc:455] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 2 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "62e54dd7499440acaa1b1c76403da552" member_type: VOTER last_known_addr { host: "127.8.87.254" port: 43115 } } peers { permanent_uuid: "4283b8cd94454e13bb77279c6afdb8f2" member_type: VOTER last_known_addr { host: "127.8.87.253" port: 46439 } } peers { permanent_uuid: "b0ae399fb47a454996698251cea1c12d" member_type: VOTER last_known_addr { host: "127.8.87.252" port: 38507 } } }
08:07:57.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:57.017124 9776 sys_catalog.cc:458] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d [sys.catalog]: This master's current role is: FOLLOWER
08:07:57.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:57.017673 9771 sys_catalog.cc:565] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d [sys.catalog]: configured and running, proceeding with master startup.
08:07:57.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:57.053789 9787 catalog_manager.cc:1269] Loaded cluster ID: 8f877e344eb84cea8a008e14c9bc53c8
08:07:57.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:57.054109 9787 catalog_manager.cc:1562] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d: loading cluster ID for follower catalog manager: success
08:07:57.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:57.061967 9787 catalog_manager.cc:1584] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d: acquiring CA information for follower catalog manager: success
08:07:57.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:57.067543 9787 catalog_manager.cc:1612] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
08:07:57.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:57.042994 9691 master_runner.cc:186] Error getting master registration for 127.8.87.254:43115: Network error: Client connection negotiation failed: client connection to 127.8.87.254:43115: connect: Connection refused (error 111),
08:07:57.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:57.068940 9691 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
08:07:57.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:57.262419 9773 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:07:57.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:57.263067 9773 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:07:57.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:57.263898 9773 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:07:57.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:57.297475 9773 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
08:07:57.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:57.297750 9773 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
08:07:57.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:57.298213 9773 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:07:57.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:57.298430 9773 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
08:07:57.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:57.298664 9773 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
08:07:57.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:57.298883 9773 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
08:07:57.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:57.299041 9773 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:40533/jwks.json
08:07:57.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:57.343879 9773 master_runner.cc:386] Master server non-default flags:
08:07:57.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:07:57.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.87.212:46335
08:07:57.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:07:57.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:07:57.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-0/data
08:07:57.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-0/wal
08:07:57.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
08:07:57.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:07:57.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.87.254:43115,127.8.87.253:46439,127.8.87.252:38507
08:07:57.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/krb5kdc/kudu.keytab
08:07:57.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.8.87.254
08:07:57.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:07:57.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:07:57.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:07:57.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.87.254:43115
08:07:57.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:07:57.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:07:57.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:07:57.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40533/jwks.json
08:07:57.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:07:57.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
08:07:57.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
08:07:57.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:07:57.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:07:57.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-0/data/info.pb
08:07:57.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.87.254
08:07:57.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33423
08:07:57.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:07:57.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:07:57.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:07:57.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:07:57.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:07:57.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-0/logs
08:07:57.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:07:57.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:07:57.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:07:57.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:07:57.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:07:57.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 18b1930e830609248ee2e26d6216d073ff40ac5d
08:07:57.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:07:57.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 23 Oct 2025 07:43:21 UTC on 5fd53c4cbb9d
08:07:57.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8677
08:07:57.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:07:57.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:57.345541 9773 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:07:57.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:57.346856 9773 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:07:57.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:57.350107 9175 heartbeater.cc:344] Connected to a master server at 127.8.87.252:38507
08:07:57.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:57.364176 9735 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "0b8b4a4be8314be4b54168e19654fe19" instance_seqno: 1761206862002646) as {username='kudu', principal='kudu/127.8.87.194@KRBTEST.COM'} at 127.8.87.194:37957; Asking this server to re-register.
08:07:57.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:57.365999 9175 heartbeater.cc:461] Registering TS with master...
08:07:57.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:57.370097 9735 ts_manager.cc:194] Registered new tserver with Master: 0b8b4a4be8314be4b54168e19654fe19 (127.8.87.194:43889)
08:07:57.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:57.375308 9797 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:07:57.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:57.565692 9025 heartbeater.cc:344] Connected to a master server at 127.8.87.252:38507
08:07:57.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:57.568589 9735 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "b89b7d622ef34a1cb032858329be126a" instance_seqno: 1761206859197829) as {username='kudu', principal='kudu/127.8.87.193@KRBTEST.COM'} at 127.8.87.193:47989; Asking this server to re-register.
08:07:57.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:57.570196 9025 heartbeater.cc:461] Registering TS with master...
08:07:57.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:57.573211 9735 ts_manager.cc:194] Registered new tserver with Master: b89b7d622ef34a1cb032858329be126a (127.8.87.193:36301)
08:07:57.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:57.890347 9322 heartbeater.cc:344] Connected to a master server at 127.8.87.252:38507
08:07:57.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:57.893352 9735 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "2c485d5a4b9f4ba6926df4b0102eed27" instance_seqno: 1761206864566991) as {username='kudu', principal='kudu/127.8.87.195@KRBTEST.COM'} at 127.8.87.195:57855; Asking this server to re-register.
08:07:57.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:57.895355 9322 heartbeater.cc:461] Registering TS with master...
08:07:57.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:57.900238 9735 ts_manager.cc:194] Registered new tserver with Master: 2c485d5a4b9f4ba6926df4b0102eed27 (127.8.87.195:36087)
08:07:58.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:58.488854 9805 raft_consensus.cc:493] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:07:58.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:58.490797 9805 raft_consensus.cc:515] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "62e54dd7499440acaa1b1c76403da552" member_type: VOTER last_known_addr { host: "127.8.87.254" port: 43115 } } peers { permanent_uuid: "4283b8cd94454e13bb77279c6afdb8f2" member_type: VOTER last_known_addr { host: "127.8.87.253" port: 46439 } } peers { permanent_uuid: "b0ae399fb47a454996698251cea1c12d" member_type: VOTER last_known_addr { host: "127.8.87.252" port: 38507 } }
08:07:58.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:58.521342 9805 leader_election.cc:290] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 62e54dd7499440acaa1b1c76403da552 (127.8.87.254:43115), 4283b8cd94454e13bb77279c6afdb8f2 (127.8.87.253:46439)
08:07:58.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:58.522922 9720 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.87.254:43115: connect: Connection refused (error 111)
08:07:58.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:58.540892 9720 leader_election.cc:336] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 62e54dd7499440acaa1b1c76403da552 (127.8.87.254:43115): Network error: Client connection negotiation failed: client connection to 127.8.87.254:43115: connect: Connection refused (error 111)
08:07:58.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:58.545074 9722 leader_election.cc:336] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 4283b8cd94454e13bb77279c6afdb8f2 (127.8.87.253:46439): Network error: Client connection negotiation failed: client connection to 127.8.87.253:46439: connect: Connection refused (error 111)
08:07:58.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:58.545722 9722 leader_election.cc:304] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: b0ae399fb47a454996698251cea1c12d; no voters: 4283b8cd94454e13bb77279c6afdb8f2, 62e54dd7499440acaa1b1c76403da552
08:07:58.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:58.551231 9805 raft_consensus.cc:2749] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d [term 2 FOLLOWER]: Leader pre-election lost for term 3. Reason: could not achieve majority
08:07:58.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 23 08:07:58 dist-test-slave-wxqn krb5kdc[8555](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761206878, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.87.193@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:07:58.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:58.921288 8907 init.cc:295] Successfully reacquired a new kerberos TGT
08:07:59.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:57.377461 9798 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:07:59.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:59.210760 9773 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 1.835s user 0.583s sys 1.189s
08:07:59.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:59.212039 9773 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 1.836s user 0.583s sys 1.189s
08:07:59.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:58.777544 9796 debug-util.cc:398] Leaking SignalData structure 0x7b0800027080 after lost signal to thread 9773
08:07:59.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:59.220606 9773 server_base.cc:1047] running on GCE node
08:07:59.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:59.221154 9812 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:07:59.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:59.222427 9773 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:07:59.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:59.225243 9773 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:07:59.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:59.226668 9773 hybrid_clock.cc:648] HybridClock initialized: now 1761206879226569 us; error 103 us; skew 500 ppm
08:07:59.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:59.227581 9773 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:07:59.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 23 08:07:59 dist-test-slave-wxqn krb5kdc[8555](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761206879, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.87.254@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:07:59.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:59.233836 9773 init.cc:377] Logged in from keytab as kudu/127.8.87.254@KRBTEST.COM (short username kudu)
08:07:59.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:59.239485 9773 webserver.cc:492] Webserver started at http://127.8.87.254:33423/ using document root <none> and password file <none>
08:07:59.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:59.240442 9773 fs_manager.cc:362] Metadata directory not provided
08:07:59.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:59.240715 9773 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:07:59.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:59.248975 9773 fs_manager.cc:714] Time spent opening directory manager: real 0.006s user 0.004s sys 0.003s
08:07:59.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:59.258869 9818 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:07:59.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:59.260139 9773 fs_manager.cc:730] Time spent opening block manager: real 0.009s user 0.003s sys 0.007s
08:07:59.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:59.260545 9773 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-0/data,/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-0/wal
08:07:59.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "62e54dd7499440acaa1b1c76403da552"
08:07:59.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-23 08:07:29 on dist-test-slave-wxqn"
08:07:59.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:59.262704 9773 fs_report.cc:389] FS layout report
08:07:59.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:07:59.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-0/wal
08:07:59.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-0/wal
08:07:59.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-0/data/data
08:07:59.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:07:59.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:07:59.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:07:59.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:07:59.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:07:59.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:07:59.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:07:59.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:07:59.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:07:59.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:07:59.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:07:59.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:59.328274 9773 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:07:59.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:59.334223 9773 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:07:59.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:59.334722 9773 kserver.cc:163] Server-wide thread pool size limit: 3276
08:07:59.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:59.421322 9773 rpc_server.cc:307] RPC server started. Bound to: 127.8.87.254:43115
08:07:59.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:59.421449 9871 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.8.87.254:43115 every 8 connection(s)
08:07:59.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:59.424016 9773 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-0/data/info.pb
08:07:59.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:59.425510 8543 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskcrgkbR/build/tsan/bin/kudu as pid 9773
08:07:59.426 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:07:59.426 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.8.87.253:46439
08:07:59.427 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
08:07:59.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:59.429054 8543 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskcrgkbR/build/tsan/bin/kudu
08:07:59.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskcrgkbR/build/tsan/bin/kudu
08:07:59.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-1/wal
08:07:59.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-1/data
08:07:59.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:07:59.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:07:59.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:07:59.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:07:59.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:07:59.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:07:59.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-1/logs
08:07:59.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-1/data/info.pb
08:07:59.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:07:59.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:07:59.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:07:59.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:07:59.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:07:59.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:07:59.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:07:59.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:07:59.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:07:59.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:07:59.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:07:59.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:07:59.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.87.253:46439
08:07:59.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.87.253
08:07:59.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44589
08:07:59.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/krb5kdc/kudu.keytab
08:07:59.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.8.87.253
08:07:59.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:07:59.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
08:07:59.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
08:07:59.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
08:07:59.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.87.212:46335
08:07:59.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:07:59.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:07:59.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:07:59.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:07:59.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.87.254:43115,127.8.87.253:46439,127.8.87.252:38507
08:07:59.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:07:59.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40533/jwks.json
08:07:59.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
08:07:59.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
08:07:59.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:59.439079 9872 sys_catalog.cc:263] Verifying existing consensus state
08:07:59.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:59.446589 9872 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 62e54dd7499440acaa1b1c76403da552: Bootstrap starting.
08:07:59.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:59.485031 9872 log.cc:826] T 00000000000000000000000000000000 P 62e54dd7499440acaa1b1c76403da552: Log is configured to *not* fsync() on all Append() calls
08:07:59.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:59.501282 9872 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 62e54dd7499440acaa1b1c76403da552: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
08:07:59.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:59.502009 9872 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 62e54dd7499440acaa1b1c76403da552: Bootstrap complete.
08:07:59.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:59.523578 9872 raft_consensus.cc:359] T 00000000000000000000000000000000 P 62e54dd7499440acaa1b1c76403da552 [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "62e54dd7499440acaa1b1c76403da552" member_type: VOTER last_known_addr { host: "127.8.87.254" port: 43115 } } peers { permanent_uuid: "4283b8cd94454e13bb77279c6afdb8f2" member_type: VOTER last_known_addr { host: "127.8.87.253" port: 46439 } } peers { permanent_uuid: "b0ae399fb47a454996698251cea1c12d" member_type: VOTER last_known_addr { host: "127.8.87.252" port: 38507 } }
08:07:59.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:59.525668 9872 raft_consensus.cc:740] T 00000000000000000000000000000000 P 62e54dd7499440acaa1b1c76403da552 [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: 62e54dd7499440acaa1b1c76403da552, State: Initialized, Role: FOLLOWER
08:07:59.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:59.526356 9872 consensus_queue.cc:260] T 00000000000000000000000000000000 P 62e54dd7499440acaa1b1c76403da552 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 2.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "62e54dd7499440acaa1b1c76403da552" member_type: VOTER last_known_addr { host: "127.8.87.254" port: 43115 } } peers { permanent_uuid: "4283b8cd94454e13bb77279c6afdb8f2" member_type: VOTER last_known_addr { host: "127.8.87.253" port: 46439 } } peers { permanent_uuid: "b0ae399fb47a454996698251cea1c12d" member_type: VOTER last_known_addr { host: "127.8.87.252" port: 38507 } }
08:07:59.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:59.529299 9876 sys_catalog.cc:455] T 00000000000000000000000000000000 P 62e54dd7499440acaa1b1c76403da552 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 2 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "62e54dd7499440acaa1b1c76403da552" member_type: VOTER last_known_addr { host: "127.8.87.254" port: 43115 } } peers { permanent_uuid: "4283b8cd94454e13bb77279c6afdb8f2" member_type: VOTER last_known_addr { host: "127.8.87.253" port: 46439 } } peers { permanent_uuid: "b0ae399fb47a454996698251cea1c12d" member_type: VOTER last_known_addr { host: "127.8.87.252" port: 38507 } } }
08:07:59.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:59.530082 9876 sys_catalog.cc:458] T 00000000000000000000000000000000 P 62e54dd7499440acaa1b1c76403da552 [sys.catalog]: This master's current role is: FOLLOWER
08:07:59.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:59.530982 9872 sys_catalog.cc:565] T 00000000000000000000000000000000 P 62e54dd7499440acaa1b1c76403da552 [sys.catalog]: configured and running, proceeding with master startup.
08:07:59.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:59.564859 9887 catalog_manager.cc:1269] Loaded cluster ID: 8f877e344eb84cea8a008e14c9bc53c8
08:07:59.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:59.565155 9887 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 62e54dd7499440acaa1b1c76403da552: loading cluster ID for follower catalog manager: success
08:07:59.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:59.572391 9887 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 62e54dd7499440acaa1b1c76403da552: acquiring CA information for follower catalog manager: success
08:07:59.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 23 08:07:59 dist-test-slave-wxqn krb5kdc[8555](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761206879, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.87.254@KRBTEST.COM for kudu/127.8.87.254@KRBTEST.COM
08:07:59.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:59.578300 9887 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 62e54dd7499440acaa1b1c76403da552: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
08:07:59.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:59.599872 9773 master_runner.cc:186] Error getting master registration for 127.8.87.253:46439: Network error: Client connection negotiation failed: client connection to 127.8.87.253:46439: connect: Connection refused (error 111),
08:07:59.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:59.627364 9773 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
08:07:59.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:59.655733 9029 heartbeater.cc:344] Connected to a master server at 127.8.87.254:43115
08:07:59.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:59.662142 9837 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "b89b7d622ef34a1cb032858329be126a" instance_seqno: 1761206859197829) as {username='kudu', principal='kudu/127.8.87.193@KRBTEST.COM'} at 127.8.87.193:37357; Asking this server to re-register.
08:07:59.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:59.663885 9029 heartbeater.cc:461] Registering TS with master...
08:07:59.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:59.668519 9837 ts_manager.cc:194] Registered new tserver with Master: b89b7d622ef34a1cb032858329be126a (127.8.87.193:36301)
08:07:59.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:59.813822 9874 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:07:59.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:59.815557 9874 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:07:59.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:59.816690 9874 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:07:59.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:59.866552 9874 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
08:07:59.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:59.866794 9874 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
08:07:59.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:59.867061 9874 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:07:59.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:59.867254 9874 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
08:07:59.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:59.867444 9874 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
08:07:59.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:59.867632 9874 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
08:07:59.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:59.867808 9874 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:40533/jwks.json
08:07:59.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:59.919498 9874 master_runner.cc:386] Master server non-default flags:
08:07:59.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:07:59.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.87.212:46335
08:07:59.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:07:59.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:07:59.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-1/data
08:07:59.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-1/wal
08:07:59.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
08:07:59.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:07:59.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.87.254:43115,127.8.87.253:46439,127.8.87.252:38507
08:07:59.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/krb5kdc/kudu.keytab
08:07:59.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.8.87.253
08:07:59.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:07:59.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:07:59.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:07:59.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.87.253:46439
08:07:59.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:07:59.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:07:59.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:07:59.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40533/jwks.json
08:07:59.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:07:59.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
08:07:59.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
08:07:59.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:07:59.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:07:59.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-1/data/info.pb
08:07:59.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.87.253
08:07:59.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44589
08:07:59.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:07:59.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:07:59.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:07:59.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:07:59.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:07:59.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-1/logs
08:07:59.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:07:59.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:07:59.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:07:59.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:07:59.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:07:59.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 18b1930e830609248ee2e26d6216d073ff40ac5d
08:07:59.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:07:59.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 23 Oct 2025 07:43:21 UTC on 5fd53c4cbb9d
08:07:59.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8677
08:07:59.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:07:59.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:59.921222 9874 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:07:59.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:59.922513 9874 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:07:59.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:59.935261 9323 heartbeater.cc:344] Connected to a master server at 127.8.87.254:43115
08:07:59.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:59.938787 9837 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "2c485d5a4b9f4ba6926df4b0102eed27" instance_seqno: 1761206864566991) as {username='kudu', principal='kudu/127.8.87.195@KRBTEST.COM'} at 127.8.87.195:43377; Asking this server to re-register.
08:07:59.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:59.940263 9323 heartbeater.cc:461] Registering TS with master...
08:07:59.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:59.943825 9837 ts_manager.cc:194] Registered new tserver with Master: 2c485d5a4b9f4ba6926df4b0102eed27 (127.8.87.195:36087)
08:07:59.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:59.945977 9900 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:08:00.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:00.263721 9906 raft_consensus.cc:493] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:08:00.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:00.264647 9906 raft_consensus.cc:515] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "62e54dd7499440acaa1b1c76403da552" member_type: VOTER last_known_addr { host: "127.8.87.254" port: 43115 } } peers { permanent_uuid: "4283b8cd94454e13bb77279c6afdb8f2" member_type: VOTER last_known_addr { host: "127.8.87.253" port: 46439 } } peers { permanent_uuid: "b0ae399fb47a454996698251cea1c12d" member_type: VOTER last_known_addr { host: "127.8.87.252" port: 38507 } }
08:08:00.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:00.281497 9906 leader_election.cc:290] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 62e54dd7499440acaa1b1c76403da552 (127.8.87.254:43115), 4283b8cd94454e13bb77279c6afdb8f2 (127.8.87.253:46439)
08:08:00.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:00.293372 9847 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "b0ae399fb47a454996698251cea1c12d" candidate_term: 3 candidate_status { last_received { term: 2 index: 5 } } ignore_live_leader: false dest_uuid: "62e54dd7499440acaa1b1c76403da552" is_pre_election: true
08:08:00.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:00.294320 9847 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 62e54dd7499440acaa1b1c76403da552 [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate b0ae399fb47a454996698251cea1c12d in term 2.
08:08:00.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:00.296173 9720 leader_election.cc:304] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 62e54dd7499440acaa1b1c76403da552, b0ae399fb47a454996698251cea1c12d; no voters:
08:08:00.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:00.297756 9906 raft_consensus.cc:2804] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d [term 2 FOLLOWER]: Leader pre-election won for term 3
08:08:00.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:00.298223 9906 raft_consensus.cc:493] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d [term 2 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
08:08:00.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:00.298987 9906 raft_consensus.cc:3060] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d [term 2 FOLLOWER]: Advancing to term 3
08:08:00.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:08:00.303068 9722 leader_election.cc:336] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 4283b8cd94454e13bb77279c6afdb8f2 (127.8.87.253:46439): Network error: Client connection negotiation failed: client connection to 127.8.87.253:46439: connect: Connection refused (error 111)
08:08:00.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:00.308965 9906 raft_consensus.cc:515] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "62e54dd7499440acaa1b1c76403da552" member_type: VOTER last_known_addr { host: "127.8.87.254" port: 43115 } } peers { permanent_uuid: "4283b8cd94454e13bb77279c6afdb8f2" member_type: VOTER last_known_addr { host: "127.8.87.253" port: 46439 } } peers { permanent_uuid: "b0ae399fb47a454996698251cea1c12d" member_type: VOTER last_known_addr { host: "127.8.87.252" port: 38507 } }
08:08:00.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:00.311550 9847 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "b0ae399fb47a454996698251cea1c12d" candidate_term: 3 candidate_status { last_received { term: 2 index: 5 } } ignore_live_leader: false dest_uuid: "62e54dd7499440acaa1b1c76403da552"
08:08:00.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:00.312107 9847 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 62e54dd7499440acaa1b1c76403da552 [term 2 FOLLOWER]: Advancing to term 3
08:08:00.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:00.312239 9906 leader_election.cc:290] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d [CANDIDATE]: Term 3 election: Requested vote from peers 62e54dd7499440acaa1b1c76403da552 (127.8.87.254:43115), 4283b8cd94454e13bb77279c6afdb8f2 (127.8.87.253:46439)
08:08:00.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:08:00.320369 9722 leader_election.cc:336] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d [CANDIDATE]: Term 3 election: RPC error from VoteRequest() call to peer 4283b8cd94454e13bb77279c6afdb8f2 (127.8.87.253:46439): Network error: Client connection negotiation failed: client connection to 127.8.87.253:46439: connect: Connection refused (error 111)
08:08:00.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:00.321486 9847 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 62e54dd7499440acaa1b1c76403da552 [term 3 FOLLOWER]: Leader election vote request: Granting yes vote for candidate b0ae399fb47a454996698251cea1c12d in term 3.
08:08:00.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:00.322748 9720 leader_election.cc:304] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d [CANDIDATE]: Term 3 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 62e54dd7499440acaa1b1c76403da552, b0ae399fb47a454996698251cea1c12d; no voters: 4283b8cd94454e13bb77279c6afdb8f2
08:08:00.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:00.323841 9906 raft_consensus.cc:2804] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d [term 3 FOLLOWER]: Leader election won for term 3
08:08:00.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:00.326050 9906 raft_consensus.cc:697] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d [term 3 LEADER]: Becoming Leader. State: Replica: b0ae399fb47a454996698251cea1c12d, State: Running, Role: LEADER
08:08:00.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:00.327168 9906 consensus_queue.cc:237] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 2.5, Last appended by leader: 5, Current term: 3, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "62e54dd7499440acaa1b1c76403da552" member_type: VOTER last_known_addr { host: "127.8.87.254" port: 43115 } } peers { permanent_uuid: "4283b8cd94454e13bb77279c6afdb8f2" member_type: VOTER last_known_addr { host: "127.8.87.253" port: 46439 } } peers { permanent_uuid: "b0ae399fb47a454996698251cea1c12d" member_type: VOTER last_known_addr { host: "127.8.87.252" port: 38507 } }
08:08:00.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:00.350118 9910 sys_catalog.cc:455] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d [sys.catalog]: SysCatalogTable state changed. Reason: New leader b0ae399fb47a454996698251cea1c12d. Latest consensus state: current_term: 3 leader_uuid: "b0ae399fb47a454996698251cea1c12d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "62e54dd7499440acaa1b1c76403da552" member_type: VOTER last_known_addr { host: "127.8.87.254" port: 43115 } } peers { permanent_uuid: "4283b8cd94454e13bb77279c6afdb8f2" member_type: VOTER last_known_addr { host: "127.8.87.253" port: 46439 } } peers { permanent_uuid: "b0ae399fb47a454996698251cea1c12d" member_type: VOTER last_known_addr { host: "127.8.87.252" port: 38507 } } }
08:08:00.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:00.351667 9910 sys_catalog.cc:458] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d [sys.catalog]: This master's current role is: LEADER
08:08:00.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:00.355990 9913 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
08:08:00.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:00.369832 9913 catalog_manager.cc:1494] Initializing Kudu cluster ID...
08:08:00.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:00.373348 9176 heartbeater.cc:344] Connected to a master server at 127.8.87.254:43115
08:08:00.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:00.376224 9837 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "0b8b4a4be8314be4b54168e19654fe19" instance_seqno: 1761206862002646) as {username='kudu', principal='kudu/127.8.87.194@KRBTEST.COM'} at 127.8.87.194:40109; Asking this server to re-register.
08:08:00.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:00.378316 9176 heartbeater.cc:461] Registering TS with master...
08:08:00.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:00.381450 9837 ts_manager.cc:194] Registered new tserver with Master: 0b8b4a4be8314be4b54168e19654fe19 (127.8.87.194:43889)
08:08:00.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:00.386088 9913 catalog_manager.cc:1269] Loaded cluster ID: 8f877e344eb84cea8a008e14c9bc53c8
08:08:00.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:00.386490 9913 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
08:08:00.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:00.392812 9913 catalog_manager.cc:1514] Loading token signing keys...
08:08:00.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:00.398197 9913 catalog_manager.cc:6033] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d: Loaded TSK: 0
08:08:00.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:00.400151 9913 catalog_manager.cc:1524] Initializing in-progress tserver states...
08:08:00.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:08:00.761976 9722 consensus_peers.cc:597] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d -> Peer 4283b8cd94454e13bb77279c6afdb8f2 (127.8.87.253:46439): Couldn't send request to peer 4283b8cd94454e13bb77279c6afdb8f2. Status: Network error: Client connection negotiation failed: client connection to 127.8.87.253:46439: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
08:08:00.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:08:00.775190 8047 debug-util.cc:398] Leaking SignalData structure 0x7b080008b420 after lost signal to thread 7977
08:08:00.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:08:00.776336 8047 debug-util.cc:398] Leaking SignalData structure 0x7b08000ae000 after lost signal to thread 8050
08:08:00.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:00.885457 9847 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 62e54dd7499440acaa1b1c76403da552 [term 3 FOLLOWER]: Refusing update from remote peer b0ae399fb47a454996698251cea1c12d: Log matching property violated. Preceding OpId in replica: term: 2 index: 5. Preceding OpId from leader: term: 3 index: 6. (index mismatch)
08:08:00.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:00.887295 9910 consensus_queue.cc:1048] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d [LEADER]: Connected to new peer: Peer: permanent_uuid: "62e54dd7499440acaa1b1c76403da552" member_type: VOTER last_known_addr { host: "127.8.87.254" port: 43115 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.001s
08:08:00.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:00.923592 9916 sys_catalog.cc:455] T 00000000000000000000000000000000 P 62e54dd7499440acaa1b1c76403da552 [sys.catalog]: SysCatalogTable state changed. Reason: New leader b0ae399fb47a454996698251cea1c12d. Latest consensus state: current_term: 3 leader_uuid: "b0ae399fb47a454996698251cea1c12d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "62e54dd7499440acaa1b1c76403da552" member_type: VOTER last_known_addr { host: "127.8.87.254" port: 43115 } } peers { permanent_uuid: "4283b8cd94454e13bb77279c6afdb8f2" member_type: VOTER last_known_addr { host: "127.8.87.253" port: 46439 } } peers { permanent_uuid: "b0ae399fb47a454996698251cea1c12d" member_type: VOTER last_known_addr { host: "127.8.87.252" port: 38507 } } }
08:08:00.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:00.924922 9916 sys_catalog.cc:458] T 00000000000000000000000000000000 P 62e54dd7499440acaa1b1c76403da552 [sys.catalog]: This master's current role is: FOLLOWER
08:08:00.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:00.945312 9916 sys_catalog.cc:455] T 00000000000000000000000000000000 P 62e54dd7499440acaa1b1c76403da552 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "b0ae399fb47a454996698251cea1c12d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "62e54dd7499440acaa1b1c76403da552" member_type: VOTER last_known_addr { host: "127.8.87.254" port: 43115 } } peers { permanent_uuid: "4283b8cd94454e13bb77279c6afdb8f2" member_type: VOTER last_known_addr { host: "127.8.87.253" port: 46439 } } peers { permanent_uuid: "b0ae399fb47a454996698251cea1c12d" member_type: VOTER last_known_addr { host: "127.8.87.252" port: 38507 } } }
08:08:00.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:00.946014 9916 sys_catalog.cc:458] T 00000000000000000000000000000000 P 62e54dd7499440acaa1b1c76403da552 [sys.catalog]: This master's current role is: FOLLOWER
08:08:00.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:00.959472 9910 sys_catalog.cc:455] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 3 leader_uuid: "b0ae399fb47a454996698251cea1c12d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "62e54dd7499440acaa1b1c76403da552" member_type: VOTER last_known_addr { host: "127.8.87.254" port: 43115 } } peers { permanent_uuid: "4283b8cd94454e13bb77279c6afdb8f2" member_type: VOTER last_known_addr { host: "127.8.87.253" port: 46439 } } peers { permanent_uuid: "b0ae399fb47a454996698251cea1c12d" member_type: VOTER last_known_addr { host: "127.8.87.252" port: 38507 } } }
08:08:00.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:00.960572 9910 sys_catalog.cc:458] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d [sys.catalog]: This master's current role is: LEADER
08:08:01.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:07:59.951730 9874 server_base.cc:1047] running on GCE node
08:08:01.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:59.952886 9903 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:08:01.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:07:59.950738 9901 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:08:01.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:01.284641 9874 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:08:01.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:01.288441 9874 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:08:01.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:01.289927 9874 hybrid_clock.cc:648] HybridClock initialized: now 1761206881289859 us; error 72 us; skew 500 ppm
08:08:01.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:01.290731 9874 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:08:01.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 23 08:08:01 dist-test-slave-wxqn krb5kdc[8555](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761206881, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.87.253@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:08:01.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:01.298596 9874 init.cc:377] Logged in from keytab as kudu/127.8.87.253@KRBTEST.COM (short username kudu)
08:08:01.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:01.305415 9874 webserver.cc:492] Webserver started at http://127.8.87.253:44589/ using document root <none> and password file <none>
08:08:01.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:01.306510 9874 fs_manager.cc:362] Metadata directory not provided
08:08:01.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:01.306761 9874 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:08:01.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:01.315369 9874 fs_manager.cc:714] Time spent opening directory manager: real 0.006s user 0.001s sys 0.004s
08:08:01.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:01.320801 9930 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:08:01.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:01.321899 9874 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
08:08:01.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:01.322185 9874 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-1/data,/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-1/wal
08:08:01.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4283b8cd94454e13bb77279c6afdb8f2"
08:08:01.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-23 08:07:31 on dist-test-slave-wxqn"
08:08:01.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:01.324152 9874 fs_report.cc:389] FS layout report
08:08:01.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:08:01.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-1/wal
08:08:01.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-1/wal
08:08:01.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-1/data/data
08:08:01.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:08:01.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:08:01.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:08:01.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:08:01.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:08:01.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:08:01.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:08:01.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:08:01.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:08:01.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:08:01.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:08:01.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:01.377517 9874 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:08:01.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:01.384282 9874 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:08:01.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:01.384761 9874 kserver.cc:163] Server-wide thread pool size limit: 3276
08:08:01.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:01.535202 9874 rpc_server.cc:307] RPC server started. Bound to: 127.8.87.253:46439
08:08:01.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:01.535392 9982 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.8.87.253:46439 every 8 connection(s)
08:08:01.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:01.538112 9874 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-1/data/info.pb
08:08:01.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:01.538892 8543 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskcrgkbR/build/tsan/bin/kudu as pid 9874
08:08:01.539 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:08:01.541 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.8.87.252:38507(127.8.87.252:38507)] connecting to peer
08:08:01.543 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.8.87.252:38507(127.8.87.252:38507)] Successfully connected to peer
08:08:01.556 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.8.87.252
08:08:01.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:01.556377 9983 sys_catalog.cc:263] Verifying existing consensus state
08:08:01.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:01.564257 9983 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4283b8cd94454e13bb77279c6afdb8f2: Bootstrap starting.
08:08:01.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:01.588639 9025 heartbeater.cc:499] Master 127.8.87.252:38507 was elected leader, sending a full tablet report...
08:08:01.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 23 08:08:01 dist-test-slave-wxqn krb5kdc[8555](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761206874, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.8.87.252@KRBTEST.COM
08:08:01.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:01.622095 9983 log.cc:826] T 00000000000000000000000000000000 P 4283b8cd94454e13bb77279c6afdb8f2: Log is configured to *not* fsync() on all Append() calls
08:08:01.635 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0xd030101c, L:/127.0.0.1:50582 - R:/127.8.87.252:38507] using SASL/GSSAPI
08:08:01.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:01.651263 9983 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4283b8cd94454e13bb77279c6afdb8f2: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
08:08:01.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:01.652338 9983 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4283b8cd94454e13bb77279c6afdb8f2: Bootstrap complete.
08:08:01.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:01.682708 9983 raft_consensus.cc:359] T 00000000000000000000000000000000 P 4283b8cd94454e13bb77279c6afdb8f2 [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "62e54dd7499440acaa1b1c76403da552" member_type: VOTER last_known_addr { host: "127.8.87.254" port: 43115 } } peers { permanent_uuid: "4283b8cd94454e13bb77279c6afdb8f2" member_type: VOTER last_known_addr { host: "127.8.87.253" port: 46439 } } peers { permanent_uuid: "b0ae399fb47a454996698251cea1c12d" member_type: VOTER last_known_addr { host: "127.8.87.252" port: 38507 } }
08:08:01.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:01.685111 9983 raft_consensus.cc:740] T 00000000000000000000000000000000 P 4283b8cd94454e13bb77279c6afdb8f2 [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4283b8cd94454e13bb77279c6afdb8f2, State: Initialized, Role: FOLLOWER
08:08:01.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:01.685912 9983 consensus_queue.cc:260] T 00000000000000000000000000000000 P 4283b8cd94454e13bb77279c6afdb8f2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 2.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "62e54dd7499440acaa1b1c76403da552" member_type: VOTER last_known_addr { host: "127.8.87.254" port: 43115 } } peers { permanent_uuid: "4283b8cd94454e13bb77279c6afdb8f2" member_type: VOTER last_known_addr { host: "127.8.87.253" port: 46439 } } peers { permanent_uuid: "b0ae399fb47a454996698251cea1c12d" member_type: VOTER last_known_addr { host: "127.8.87.252" port: 38507 } }
08:08:01.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:01.688760 9987 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4283b8cd94454e13bb77279c6afdb8f2 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 2 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "62e54dd7499440acaa1b1c76403da552" member_type: VOTER last_known_addr { host: "127.8.87.254" port: 43115 } } peers { permanent_uuid: "4283b8cd94454e13bb77279c6afdb8f2" member_type: VOTER last_known_addr { host: "127.8.87.253" port: 46439 } } peers { permanent_uuid: "b0ae399fb47a454996698251cea1c12d" member_type: VOTER last_known_addr { host: "127.8.87.252" port: 38507 } } }
08:08:01.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:01.689597 9987 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4283b8cd94454e13bb77279c6afdb8f2 [sys.catalog]: This master's current role is: FOLLOWER
08:08:01.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:01.690583 9983 sys_catalog.cc:565] T 00000000000000000000000000000000 P 4283b8cd94454e13bb77279c6afdb8f2 [sys.catalog]: configured and running, proceeding with master startup.
08:08:01.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:01.732064 9998 catalog_manager.cc:1269] Loaded cluster ID: 8f877e344eb84cea8a008e14c9bc53c8
08:08:01.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:01.732324 9998 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 4283b8cd94454e13bb77279c6afdb8f2: loading cluster ID for follower catalog manager: success
08:08:01.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 23 08:08:01 dist-test-slave-wxqn krb5kdc[8555](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761206881, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.87.194@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:08:01.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:01.740345 9056 init.cc:295] Successfully reacquired a new kerberos TGT
08:08:01.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:01.751667 9998 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 4283b8cd94454e13bb77279c6afdb8f2: acquiring CA information for follower catalog manager: success
08:08:01.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:01.765410 9958 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 4283b8cd94454e13bb77279c6afdb8f2 [term 2 FOLLOWER]: Advancing to term 3
08:08:01.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:01.766484 9998 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 4283b8cd94454e13bb77279c6afdb8f2: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
08:08:01.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 23 08:08:01 dist-test-slave-wxqn krb5kdc[8555](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761206881, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.87.253@KRBTEST.COM for kudu/127.8.87.254@KRBTEST.COM
08:08:01.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:01.785254 9024 heartbeater.cc:344] Connected to a master server at 127.8.87.253:46439
08:08:01.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:01.865288 9987 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4283b8cd94454e13bb77279c6afdb8f2 [sys.catalog]: SysCatalogTable state changed. Reason: New leader b0ae399fb47a454996698251cea1c12d. Latest consensus state: current_term: 3 leader_uuid: "b0ae399fb47a454996698251cea1c12d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "62e54dd7499440acaa1b1c76403da552" member_type: VOTER last_known_addr { host: "127.8.87.254" port: 43115 } } peers { permanent_uuid: "4283b8cd94454e13bb77279c6afdb8f2" member_type: VOTER last_known_addr { host: "127.8.87.253" port: 46439 } } peers { permanent_uuid: "b0ae399fb47a454996698251cea1c12d" member_type: VOTER last_known_addr { host: "127.8.87.252" port: 38507 } } }
08:08:01.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:01.865937 9987 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4283b8cd94454e13bb77279c6afdb8f2 [sys.catalog]: This master's current role is: FOLLOWER
08:08:01.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:01.873833 10007 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4283b8cd94454e13bb77279c6afdb8f2 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "b0ae399fb47a454996698251cea1c12d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "62e54dd7499440acaa1b1c76403da552" member_type: VOTER last_known_addr { host: "127.8.87.254" port: 43115 } } peers { permanent_uuid: "4283b8cd94454e13bb77279c6afdb8f2" member_type: VOTER last_known_addr { host: "127.8.87.253" port: 46439 } } peers { permanent_uuid: "b0ae399fb47a454996698251cea1c12d" member_type: VOTER last_known_addr { host: "127.8.87.252" port: 38507 } } }
08:08:01.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:01.874569 10007 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4283b8cd94454e13bb77279c6afdb8f2 [sys.catalog]: This master's current role is: FOLLOWER
08:08:01.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:01.874646 9947 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "b89b7d622ef34a1cb032858329be126a" instance_seqno: 1761206859197829) as {username='kudu', principal='kudu/127.8.87.193@KRBTEST.COM'} at 127.8.87.193:37323; Asking this server to re-register.
08:08:01.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:01.876235 9024 heartbeater.cc:461] Registering TS with master...
08:08:01.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:01.880045 9946 ts_manager.cc:194] Registered new tserver with Master: b89b7d622ef34a1cb032858329be126a (127.8.87.193:36301)
08:08:01.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:01.915187 9322 heartbeater.cc:499] Master 127.8.87.252:38507 was elected leader, sending a full tablet report...
08:08:01.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:01.983074 9321 heartbeater.cc:344] Connected to a master server at 127.8.87.253:46439
08:08:01.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:01.985882 9947 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "2c485d5a4b9f4ba6926df4b0102eed27" instance_seqno: 1761206864566991) as {username='kudu', principal='kudu/127.8.87.195@KRBTEST.COM'} at 127.8.87.195:42361; Asking this server to re-register.
08:08:01.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:01.987149 9321 heartbeater.cc:461] Registering TS with master...
08:08:01.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:01.991058 9947 ts_manager.cc:194] Registered new tserver with Master: 2c485d5a4b9f4ba6926df4b0102eed27 (127.8.87.195:36087)
08:08:02.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:02.233520 9910 sys_catalog.cc:455] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 3 leader_uuid: "b0ae399fb47a454996698251cea1c12d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "62e54dd7499440acaa1b1c76403da552" member_type: VOTER last_known_addr { host: "127.8.87.254" port: 43115 } } peers { permanent_uuid: "4283b8cd94454e13bb77279c6afdb8f2" member_type: VOTER last_known_addr { host: "127.8.87.253" port: 46439 } } peers { permanent_uuid: "b0ae399fb47a454996698251cea1c12d" member_type: VOTER last_known_addr { host: "127.8.87.252" port: 38507 } } }
08:08:02.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:02.237087 9910 sys_catalog.cc:458] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d [sys.catalog]: This master's current role is: LEADER
08:08:02.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:02.389990 9175 heartbeater.cc:499] Master 127.8.87.252:38507 was elected leader, sending a full tablet report...
08:08:02.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:02.489643 9174 heartbeater.cc:344] Connected to a master server at 127.8.87.253:46439
08:08:02.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:02.492354 9947 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "0b8b4a4be8314be4b54168e19654fe19" instance_seqno: 1761206862002646) as {username='kudu', principal='kudu/127.8.87.194@KRBTEST.COM'} at 127.8.87.194:59897; Asking this server to re-register.
08:08:02.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:02.493614 9174 heartbeater.cc:461] Registering TS with master...
08:08:02.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:02.496209 9947 ts_manager.cc:194] Registered new tserver with Master: 0b8b4a4be8314be4b54168e19654fe19 (127.8.87.194:43889)
08:08:02.644 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
08:08:02.644 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
08:08:02.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 23 08:08:02 dist-test-slave-wxqn krb5kdc[8555](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761206882, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:08:02.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
08:08:02.683 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:08:02.683 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/krb5kdc/krb5cc
08:08:02.685 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
08:08:02.685 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.8.87.252:38507
08:08:02.685 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
08:08:02.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:02.685672 8543 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskcrgkbR/build/tsan/bin/kudu with pid 9691
08:08:02.717 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.8.87.252:38507(127.8.87.252:38507)] handling channelInactive
08:08:02.717 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.87.252:38507(127.8.87.252:38507)] cleaning up while in state READY due to: connection closed
08:08:02.718 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:08:02.718 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.8.87.254:43115
08:08:02.719 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
08:08:02.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:02.719319 8543 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskcrgkbR/build/tsan/bin/kudu with pid 9773
08:08:02.752 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:08:02.752 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.8.87.253:46439
08:08:02.753 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
08:08:02.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:02.753468 8543 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskcrgkbR/build/tsan/bin/kudu with pid 9874
08:08:02.783 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:08:02.783 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.8.87.252:38507
08:08:02.784 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
08:08:02.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:02.785292 8543 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskcrgkbR/build/tsan/bin/kudu
08:08:02.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskcrgkbR/build/tsan/bin/kudu
08:08:02.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-2/wal
08:08:02.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-2/data
08:08:02.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:08:02.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:08:02.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:08:02.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:08:02.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:08:02.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:08:02.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-2/logs
08:08:02.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-2/data/info.pb
08:08:02.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:08:02.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:08:02.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:08:02.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:08:02.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:08:02.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:08:02.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:08:02.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:08:02.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:08:02.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:08:02.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:08:02.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:08:02.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.87.252:38507
08:08:02.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.87.252
08:08:02.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=40629
08:08:02.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/krb5kdc/kudu.keytab
08:08:02.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.8.87.252
08:08:02.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:08:02.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
08:08:02.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
08:08:02.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
08:08:02.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.87.212:46335
08:08:02.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:08:02.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:08:02.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:08:02.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:08:02.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.87.254:43115,127.8.87.253:46439,127.8.87.252:38507
08:08:02.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:08:02.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40533/jwks.json
08:08:02.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
08:08:02.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
08:08:03.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:08:03.118803 10018 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:08:03.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:08:03.119351 10018 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:08:03.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:08:03.119989 10018 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:08:03.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:08:03.153738 10018 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
08:08:03.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:08:03.153952 10018 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
08:08:03.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:08:03.154201 10018 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:08:03.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:08:03.154390 10018 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
08:08:03.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:08:03.154575 10018 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
08:08:03.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:08:03.154763 10018 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
08:08:03.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:08:03.154927 10018 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:40533/jwks.json
08:08:03.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:03.193233 10018 master_runner.cc:386] Master server non-default flags:
08:08:03.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:08:03.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.87.212:46335
08:08:03.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:08:03.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:08:03.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-2/data
08:08:03.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-2/wal
08:08:03.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
08:08:03.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:08:03.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.87.254:43115,127.8.87.253:46439,127.8.87.252:38507
08:08:03.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/krb5kdc/kudu.keytab
08:08:03.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.8.87.252
08:08:03.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:08:03.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:08:03.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:08:03.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.87.252:38507
08:08:03.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:08:03.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:08:03.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:08:03.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40533/jwks.json
08:08:03.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:08:03.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
08:08:03.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
08:08:03.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:08:03.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:08:03.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-2/data/info.pb
08:08:03.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.87.252
08:08:03.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=40629
08:08:03.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:08:03.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:08:03.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:08:03.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:08:03.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:08:03.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-2/logs
08:08:03.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:08:03.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:08:03.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:08:03.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:08:03.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:08:03.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 18b1930e830609248ee2e26d6216d073ff40ac5d
08:08:03.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:08:03.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 23 Oct 2025 07:43:21 UTC on 5fd53c4cbb9d
08:08:03.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8677
08:08:03.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:08:03.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:03.194476 10018 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:08:03.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:03.195452 10018 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:08:03.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:08:03.206817 10026 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:08:04.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 23 08:08:04 dist-test-slave-wxqn krb5kdc[8555](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761206884, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.87.195@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:08:04.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:04.138049 9204 init.cc:295] Successfully reacquired a new kerberos TGT
08:08:04.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:08:04.610359 10025 debug-util.cc:398] Leaking SignalData structure 0x7b0800027080 after lost signal to thread 10018
08:08:04.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:08:04.722308 10018 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.515s user 0.458s sys 0.941s
08:08:04.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:08:03.207382 10027 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:08:04.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:08:04.722821 10018 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.516s user 0.458s sys 0.941s
08:08:04.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:08:04.724597 10029 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:08:04.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:08:04.727344 10028 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1516 milliseconds
08:08:04.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:04.727419 10018 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
08:08:04.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:04.728761 10018 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:08:04.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:04.731066 10018 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:08:04.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:04.732388 10018 hybrid_clock.cc:648] HybridClock initialized: now 1761206884732358 us; error 45 us; skew 500 ppm
08:08:04.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:04.733212 10018 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:08:04.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 23 08:08:04 dist-test-slave-wxqn krb5kdc[8555](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761206884, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.87.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:08:04.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:04.739444 10018 init.cc:377] Logged in from keytab as kudu/127.8.87.252@KRBTEST.COM (short username kudu)
08:08:04.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:04.744261 10018 webserver.cc:492] Webserver started at http://127.8.87.252:40629/ using document root <none> and password file <none>
08:08:04.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:04.745187 10018 fs_manager.cc:362] Metadata directory not provided
08:08:04.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:04.745421 10018 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:08:04.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:04.753883 10018 fs_manager.cc:714] Time spent opening directory manager: real 0.006s user 0.006s sys 0.002s
08:08:04.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:04.758455 10051 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:08:04.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:04.759613 10018 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.004s sys 0.000s
08:08:04.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:04.760031 10018 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-2/data,/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-2/wal
08:08:04.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b0ae399fb47a454996698251cea1c12d"
08:08:04.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-23 08:07:34 on dist-test-slave-wxqn"
08:08:04.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:04.762286 10018 fs_report.cc:389] FS layout report
08:08:04.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:08:04.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-2/wal
08:08:04.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-2/wal
08:08:04.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-2/data/data
08:08:04.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:08:04.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:08:04.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:08:04.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:08:04.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:08:04.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:08:04.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:08:04.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:08:04.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:08:04.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:08:04.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:08:04.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:04.842893 10018 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:08:04.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:04.851318 10018 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:08:04.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:04.851876 10018 kserver.cc:163] Server-wide thread pool size limit: 3276
08:08:04.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:08:04.884639 9020 debug-util.cc:398] Leaking SignalData structure 0x7b08000ae9a0 after lost signal to thread 8891
08:08:04.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:08:04.885545 9020 debug-util.cc:398] Leaking SignalData structure 0x7b08000b3880 after lost signal to thread 9023
08:08:04.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:04.934170 10018 rpc_server.cc:307] RPC server started. Bound to: 127.8.87.252:38507
08:08:04.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:04.934283 10103 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.8.87.252:38507 every 8 connection(s)
08:08:04.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:04.938179 10018 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-2/data/info.pb
08:08:04.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:04.943905 8543 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskcrgkbR/build/tsan/bin/kudu as pid 10018
08:08:04.945 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:08:04.945 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.8.87.254:43115
08:08:04.945 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
08:08:04.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:04.946681 8543 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskcrgkbR/build/tsan/bin/kudu
08:08:04.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskcrgkbR/build/tsan/bin/kudu
08:08:04.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-0/wal
08:08:04.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-0/data
08:08:04.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:08:04.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:08:04.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:08:04.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:08:04.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:08:04.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:08:04.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-0/logs
08:08:04.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-0/data/info.pb
08:08:04.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:08:04.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:08:04.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:08:04.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:08:04.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:08:04.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:08:04.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:08:04.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:08:04.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:08:04.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:08:04.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:08:04.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:08:04.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.87.254:43115
08:08:04.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.87.254
08:08:04.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33423
08:08:04.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/krb5kdc/kudu.keytab
08:08:04.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.8.87.254
08:08:04.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:08:04.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
08:08:04.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
08:08:04.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
08:08:04.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.87.212:46335
08:08:04.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:08:04.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:08:04.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:08:04.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:08:04.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.87.254:43115,127.8.87.253:46439,127.8.87.252:38507
08:08:04.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:08:04.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40533/jwks.json
08:08:04.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
08:08:04.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
08:08:04.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:04.958629 10104 sys_catalog.cc:263] Verifying existing consensus state
08:08:04.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:04.966336 10104 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d: Bootstrap starting.
08:08:05.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 23 08:08:05 dist-test-slave-wxqn krb5kdc[8555](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761206884, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.87.195@KRBTEST.COM for kudu/127.8.87.252@KRBTEST.COM
08:08:05.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:05.036350 9322 heartbeater.cc:344] Connected to a master server at 127.8.87.252:38507
08:08:05.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:05.048331 10104 log.cc:826] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d: Log is configured to *not* fsync() on all Append() calls
08:08:05.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:05.072072 10104 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d: Bootstrap replayed 1/1 log segments. Stats: ops{read=6 overwritten=0 applied=6 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
08:08:05.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:05.073109 10104 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d: Bootstrap complete.
08:08:05.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:05.106422 10104 raft_consensus.cc:359] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d [term 3 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "62e54dd7499440acaa1b1c76403da552" member_type: VOTER last_known_addr { host: "127.8.87.254" port: 43115 } } peers { permanent_uuid: "4283b8cd94454e13bb77279c6afdb8f2" member_type: VOTER last_known_addr { host: "127.8.87.253" port: 46439 } } peers { permanent_uuid: "b0ae399fb47a454996698251cea1c12d" member_type: VOTER last_known_addr { host: "127.8.87.252" port: 38507 } }
08:08:05.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:05.110245 10104 raft_consensus.cc:740] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d [term 3 FOLLOWER]: Becoming Follower/Learner. State: Replica: b0ae399fb47a454996698251cea1c12d, State: Initialized, Role: FOLLOWER
08:08:05.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:05.111400 10104 consensus_queue.cc:260] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 6, Last appended: 3.6, Last appended by leader: 6, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "62e54dd7499440acaa1b1c76403da552" member_type: VOTER last_known_addr { host: "127.8.87.254" port: 43115 } } peers { permanent_uuid: "4283b8cd94454e13bb77279c6afdb8f2" member_type: VOTER last_known_addr { host: "127.8.87.253" port: 46439 } } peers { permanent_uuid: "b0ae399fb47a454996698251cea1c12d" member_type: VOTER last_known_addr { host: "127.8.87.252" port: 38507 } }
08:08:05.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:05.121235 10112 sys_catalog.cc:455] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 3 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "62e54dd7499440acaa1b1c76403da552" member_type: VOTER last_known_addr { host: "127.8.87.254" port: 43115 } } peers { permanent_uuid: "4283b8cd94454e13bb77279c6afdb8f2" member_type: VOTER last_known_addr { host: "127.8.87.253" port: 46439 } } peers { permanent_uuid: "b0ae399fb47a454996698251cea1c12d" member_type: VOTER last_known_addr { host: "127.8.87.252" port: 38507 } } }
08:08:05.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:05.122136 10112 sys_catalog.cc:458] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d [sys.catalog]: This master's current role is: FOLLOWER
08:08:05.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:05.125576 10104 sys_catalog.cc:565] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d [sys.catalog]: configured and running, proceeding with master startup.
08:08:05.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:05.198235 10124 catalog_manager.cc:1269] Loaded cluster ID: 8f877e344eb84cea8a008e14c9bc53c8
08:08:05.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:05.199183 10124 catalog_manager.cc:1562] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d: loading cluster ID for follower catalog manager: success
08:08:05.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:05.217119 10124 catalog_manager.cc:1584] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d: acquiring CA information for follower catalog manager: success
08:08:05.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:05.199363 10018 master_runner.cc:186] Error getting master registration for 127.8.87.254:43115: Network error: Client connection negotiation failed: client connection to 127.8.87.254:43115: connect: Connection refused (error 111),
08:08:05.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:05.225442 10018 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
08:08:05.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:05.228929 10124 catalog_manager.cc:1612] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
08:08:05.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:05.492905 9175 heartbeater.cc:344] Connected to a master server at 127.8.87.252:38507
08:08:05.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:05.499456 10069 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "0b8b4a4be8314be4b54168e19654fe19" instance_seqno: 1761206862002646) as {username='kudu', principal='kudu/127.8.87.194@KRBTEST.COM'} at 127.8.87.194:51857; Asking this server to re-register.
08:08:05.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:05.500792 9175 heartbeater.cc:461] Registering TS with master...
08:08:05.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:05.504123 10069 ts_manager.cc:194] Registered new tserver with Master: 0b8b4a4be8314be4b54168e19654fe19 (127.8.87.194:43889)
08:08:05.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:08:05.536545 10106 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:08:05.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:08:05.537611 10106 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:08:05.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:08:05.538722 10106 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:08:05.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:08:05.582338 10106 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
08:08:05.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:08:05.582700 10106 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
08:08:05.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:08:05.583040 10106 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:08:05.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:08:05.583319 10106 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
08:08:05.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:08:05.583636 10106 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
08:08:05.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:08:05.583973 10106 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
08:08:05.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:08:05.584200 10106 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:40533/jwks.json
08:08:05.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:05.634398 10106 master_runner.cc:386] Master server non-default flags:
08:08:05.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:08:05.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.87.212:46335
08:08:05.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:08:05.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:08:05.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-0/data
08:08:05.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-0/wal
08:08:05.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
08:08:05.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:08:05.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.87.254:43115,127.8.87.253:46439,127.8.87.252:38507
08:08:05.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/krb5kdc/kudu.keytab
08:08:05.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.8.87.254
08:08:05.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:08:05.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:08:05.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:08:05.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.87.254:43115
08:08:05.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:08:05.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:08:05.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:08:05.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40533/jwks.json
08:08:05.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:08:05.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
08:08:05.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
08:08:05.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:08:05.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:08:05.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-0/data/info.pb
08:08:05.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.87.254
08:08:05.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33423
08:08:05.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:08:05.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:08:05.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:08:05.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:08:05.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:08:05.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-0/logs
08:08:05.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:08:05.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:08:05.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:08:05.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:08:05.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:08:05.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 18b1930e830609248ee2e26d6216d073ff40ac5d
08:08:05.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:08:05.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 23 Oct 2025 07:43:21 UTC on 5fd53c4cbb9d
08:08:05.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8677
08:08:05.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:08:05.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:05.636165 10106 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:08:05.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:05.637315 10106 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:08:05.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:08:05.658561 10134 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:08:05.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:05.680334 9025 heartbeater.cc:344] Connected to a master server at 127.8.87.252:38507
08:08:05.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:05.684167 10069 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "b89b7d622ef34a1cb032858329be126a" instance_seqno: 1761206859197829) as {username='kudu', principal='kudu/127.8.87.193@KRBTEST.COM'} at 127.8.87.193:39503; Asking this server to re-register.
08:08:05.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:05.687539 9025 heartbeater.cc:461] Registering TS with master...
08:08:05.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:05.692909 10069 ts_manager.cc:194] Registered new tserver with Master: b89b7d622ef34a1cb032858329be126a (127.8.87.193:36301)
08:08:06.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:06.044607 10069 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "2c485d5a4b9f4ba6926df4b0102eed27" instance_seqno: 1761206864566991) as {username='kudu', principal='kudu/127.8.87.195@KRBTEST.COM'} at 127.8.87.195:52409; Asking this server to re-register.
08:08:06.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:06.048292 9322 heartbeater.cc:461] Registering TS with master...
08:08:06.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:06.055648 10069 ts_manager.cc:194] Registered new tserver with Master: 2c485d5a4b9f4ba6926df4b0102eed27 (127.8.87.195:36087)
08:08:06.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:06.671168 10148 raft_consensus.cc:493] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d [term 3 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:08:06.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:06.672400 10148 raft_consensus.cc:515] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "62e54dd7499440acaa1b1c76403da552" member_type: VOTER last_known_addr { host: "127.8.87.254" port: 43115 } } peers { permanent_uuid: "4283b8cd94454e13bb77279c6afdb8f2" member_type: VOTER last_known_addr { host: "127.8.87.253" port: 46439 } } peers { permanent_uuid: "b0ae399fb47a454996698251cea1c12d" member_type: VOTER last_known_addr { host: "127.8.87.252" port: 38507 } }
08:08:06.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:08:06.731981 10053 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.87.254:43115: connect: Connection refused (error 111)
08:08:06.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:06.738416 10148 leader_election.cc:290] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 62e54dd7499440acaa1b1c76403da552 (127.8.87.254:43115), 4283b8cd94454e13bb77279c6afdb8f2 (127.8.87.253:46439)
08:08:06.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:08:06.745496 10053 leader_election.cc:336] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 62e54dd7499440acaa1b1c76403da552 (127.8.87.254:43115): Network error: Client connection negotiation failed: client connection to 127.8.87.254:43115: connect: Connection refused (error 111)
08:08:06.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:08:06.747570 10055 leader_election.cc:336] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 4283b8cd94454e13bb77279c6afdb8f2 (127.8.87.253:46439): Network error: Client connection negotiation failed: client connection to 127.8.87.253:46439: connect: Connection refused (error 111)
08:08:06.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:06.748553 10055 leader_election.cc:304] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: b0ae399fb47a454996698251cea1c12d; no voters: 4283b8cd94454e13bb77279c6afdb8f2, 62e54dd7499440acaa1b1c76403da552
08:08:06.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:06.752369 10148 raft_consensus.cc:2749] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
08:08:07.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:08:07.710196 7966 debug-util.cc:398] Leaking SignalData structure 0x7b08000af300 after lost signal to thread 7901
08:08:07.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:08:07.711023 7966 debug-util.cc:398] Leaking SignalData structure 0x7b08000b1240 after lost signal to thread 7969
08:08:08.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:08.486464 10160 raft_consensus.cc:493] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d [term 3 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:08:08.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:08.488027 10160 raft_consensus.cc:515] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "62e54dd7499440acaa1b1c76403da552" member_type: VOTER last_known_addr { host: "127.8.87.254" port: 43115 } } peers { permanent_uuid: "4283b8cd94454e13bb77279c6afdb8f2" member_type: VOTER last_known_addr { host: "127.8.87.253" port: 46439 } } peers { permanent_uuid: "b0ae399fb47a454996698251cea1c12d" member_type: VOTER last_known_addr { host: "127.8.87.252" port: 38507 } }
08:08:08.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:08.521072 10160 leader_election.cc:290] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 62e54dd7499440acaa1b1c76403da552 (127.8.87.254:43115), 4283b8cd94454e13bb77279c6afdb8f2 (127.8.87.253:46439)
08:08:08.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:08:08.574018 10055 leader_election.cc:336] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 4283b8cd94454e13bb77279c6afdb8f2 (127.8.87.253:46439): Network error: Client connection negotiation failed: client connection to 127.8.87.253:46439: connect: Connection refused (error 111)
08:08:08.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:08:08.589581 10053 leader_election.cc:336] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 62e54dd7499440acaa1b1c76403da552 (127.8.87.254:43115): Network error: Client connection negotiation failed: client connection to 127.8.87.254:43115: connect: Connection refused (error 111)
08:08:08.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:08.591239 10053 leader_election.cc:304] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: b0ae399fb47a454996698251cea1c12d; no voters: 4283b8cd94454e13bb77279c6afdb8f2, 62e54dd7499440acaa1b1c76403da552
08:08:08.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:08.595316 10160 raft_consensus.cc:2749] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
08:08:08.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:08:07.053440 10133 debug-util.cc:398] Leaking SignalData structure 0x7b0800027080 after lost signal to thread 10106
08:08:08.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:08:08.923220 10133 kernel_stack_watchdog.cc:198] Thread 10106 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 384ms:
08:08:08.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
08:08:08.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
08:08:08.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
08:08:08.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
08:08:08.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 23 08:08:08 dist-test-slave-wxqn krb5kdc[8555](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761206888, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.87.193@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:08:08.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:08:05.668989 10136 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:08:08.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:08.924835 8907 init.cc:295] Successfully reacquired a new kerberos TGT
08:08:08.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:08:08.928023 10106 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 3.262s user 0.716s sys 1.194s
08:08:08.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:08:08.928426 10106 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 3.262s user 0.716s sys 1.194s
08:08:08.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:08:08.929268 10137 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 3257 milliseconds
08:08:08.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:08:08.929999 10139 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:08:08.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:08.930095 10106 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
08:08:08.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:08.931867 10106 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:08:08.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:08.946255 10106 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:08:08.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:08.950142 10106 hybrid_clock.cc:648] HybridClock initialized: now 1761206888950092 us; error 52 us; skew 500 ppm
08:08:08.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:08.951522 10106 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:08:08.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 23 08:08:08 dist-test-slave-wxqn krb5kdc[8555](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761206888, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.87.254@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:08:08.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:08.960845 10106 init.cc:377] Logged in from keytab as kudu/127.8.87.254@KRBTEST.COM (short username kudu)
08:08:08.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:08.973182 10106 webserver.cc:492] Webserver started at http://127.8.87.254:33423/ using document root <none> and password file <none>
08:08:08.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:08.974440 10106 fs_manager.cc:362] Metadata directory not provided
08:08:08.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:08.974731 10106 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:08:09.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:09.002772 10106 fs_manager.cc:714] Time spent opening directory manager: real 0.024s user 0.005s sys 0.005s
08:08:09.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:09.009891 10175 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:08:09.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:09.011242 10106 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.000s sys 0.003s
08:08:09.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:09.011678 10106 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-0/data,/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-0/wal
08:08:09.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "62e54dd7499440acaa1b1c76403da552"
08:08:09.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-23 08:07:29 on dist-test-slave-wxqn"
08:08:09.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:09.014387 10106 fs_report.cc:389] FS layout report
08:08:09.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:08:09.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-0/wal
08:08:09.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-0/wal
08:08:09.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-0/data/data
08:08:09.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:08:09.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:08:09.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:08:09.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:08:09.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:08:09.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:08:09.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:08:09.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:08:09.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:08:09.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:08:09.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:08:09.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:09.103832 10106 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:08:09.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:09.111773 10106 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:08:09.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:09.112318 10106 kserver.cc:163] Server-wide thread pool size limit: 3276
08:08:09.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:09.235503 10106 rpc_server.cc:307] RPC server started. Bound to: 127.8.87.254:43115
08:08:09.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:09.235705 10230 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.8.87.254:43115 every 8 connection(s)
08:08:09.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:09.239573 10106 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-0/data/info.pb
08:08:09.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:09.240921 8543 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskcrgkbR/build/tsan/bin/kudu as pid 10106
08:08:09.241 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:08:09.242 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.8.87.253:46439
08:08:09.242 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
08:08:09.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:09.244486 8543 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskcrgkbR/build/tsan/bin/kudu
08:08:09.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskcrgkbR/build/tsan/bin/kudu
08:08:09.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-1/wal
08:08:09.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-1/data
08:08:09.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:08:09.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:08:09.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:08:09.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:08:09.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:08:09.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:08:09.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-1/logs
08:08:09.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-1/data/info.pb
08:08:09.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:08:09.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:08:09.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:08:09.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:08:09.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:08:09.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:08:09.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:08:09.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:08:09.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:08:09.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:08:09.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:08:09.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:08:09.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.87.253:46439
08:08:09.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.87.253
08:08:09.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44589
08:08:09.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/krb5kdc/kudu.keytab
08:08:09.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.8.87.253
08:08:09.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:08:09.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
08:08:09.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
08:08:09.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
08:08:09.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.87.212:46335
08:08:09.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:08:09.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:08:09.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:08:09.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:08:09.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.87.254:43115,127.8.87.253:46439,127.8.87.252:38507
08:08:09.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:08:09.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40533/jwks.json
08:08:09.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
08:08:09.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
08:08:09.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:09.285234 10231 sys_catalog.cc:263] Verifying existing consensus state
08:08:09.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:09.294392 10231 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 62e54dd7499440acaa1b1c76403da552: Bootstrap starting.
08:08:09.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:09.356999 10231 log.cc:826] T 00000000000000000000000000000000 P 62e54dd7499440acaa1b1c76403da552: Log is configured to *not* fsync() on all Append() calls
08:08:09.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:09.389711 10231 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 62e54dd7499440acaa1b1c76403da552: Bootstrap replayed 1/1 log segments. Stats: ops{read=6 overwritten=0 applied=6 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
08:08:09.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:09.391053 10231 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 62e54dd7499440acaa1b1c76403da552: Bootstrap complete.
08:08:09.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:09.433248 10231 raft_consensus.cc:359] T 00000000000000000000000000000000 P 62e54dd7499440acaa1b1c76403da552 [term 3 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "62e54dd7499440acaa1b1c76403da552" member_type: VOTER last_known_addr { host: "127.8.87.254" port: 43115 } } peers { permanent_uuid: "4283b8cd94454e13bb77279c6afdb8f2" member_type: VOTER last_known_addr { host: "127.8.87.253" port: 46439 } } peers { permanent_uuid: "b0ae399fb47a454996698251cea1c12d" member_type: VOTER last_known_addr { host: "127.8.87.252" port: 38507 } }
08:08:09.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:09.438400 10231 raft_consensus.cc:740] T 00000000000000000000000000000000 P 62e54dd7499440acaa1b1c76403da552 [term 3 FOLLOWER]: Becoming Follower/Learner. State: Replica: 62e54dd7499440acaa1b1c76403da552, State: Initialized, Role: FOLLOWER
08:08:09.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:09.440042 10231 consensus_queue.cc:260] T 00000000000000000000000000000000 P 62e54dd7499440acaa1b1c76403da552 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 6, Last appended: 3.6, Last appended by leader: 6, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "62e54dd7499440acaa1b1c76403da552" member_type: VOTER last_known_addr { host: "127.8.87.254" port: 43115 } } peers { permanent_uuid: "4283b8cd94454e13bb77279c6afdb8f2" member_type: VOTER last_known_addr { host: "127.8.87.253" port: 46439 } } peers { permanent_uuid: "b0ae399fb47a454996698251cea1c12d" member_type: VOTER last_known_addr { host: "127.8.87.252" port: 38507 } }
08:08:09.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:09.453541 10231 sys_catalog.cc:565] T 00000000000000000000000000000000 P 62e54dd7499440acaa1b1c76403da552 [sys.catalog]: configured and running, proceeding with master startup.
08:08:09.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:09.458534 10236 sys_catalog.cc:455] T 00000000000000000000000000000000 P 62e54dd7499440acaa1b1c76403da552 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 3 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "62e54dd7499440acaa1b1c76403da552" member_type: VOTER last_known_addr { host: "127.8.87.254" port: 43115 } } peers { permanent_uuid: "4283b8cd94454e13bb77279c6afdb8f2" member_type: VOTER last_known_addr { host: "127.8.87.253" port: 46439 } } peers { permanent_uuid: "b0ae399fb47a454996698251cea1c12d" member_type: VOTER last_known_addr { host: "127.8.87.252" port: 38507 } } }
08:08:09.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:09.459791 10236 sys_catalog.cc:458] T 00000000000000000000000000000000 P 62e54dd7499440acaa1b1c76403da552 [sys.catalog]: This master's current role is: FOLLOWER
08:08:09.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:09.555039 10247 catalog_manager.cc:805] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
08:08:09.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:08:09.576282 9317 debug-util.cc:398] Leaking SignalData structure 0x7b080003de80 after lost signal to thread 9191
08:08:09.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:08:09.577445 9317 debug-util.cc:398] Leaking SignalData structure 0x7b08000bab20 after lost signal to thread 9320
08:08:09.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 23 08:08:09 dist-test-slave-wxqn krb5kdc[8555](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761206888, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.87.254@KRBTEST.COM for kudu/127.8.87.254@KRBTEST.COM
08:08:09.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 23 08:08:09 dist-test-slave-wxqn krb5kdc[8555](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761206881, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.87.194@KRBTEST.COM for kudu/127.8.87.254@KRBTEST.COM
08:08:09.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:09.638446 9176 heartbeater.cc:344] Connected to a master server at 127.8.87.254:43115
08:08:09.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:09.668780 10196 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "0b8b4a4be8314be4b54168e19654fe19" instance_seqno: 1761206862002646) as {username='kudu', principal='kudu/127.8.87.194@KRBTEST.COM'} at 127.8.87.194:48407; Asking this server to re-register.
08:08:09.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:09.631891 10106 master_runner.cc:186] Error getting master registration for 127.8.87.253:46439: Network error: Client connection negotiation failed: client connection to 127.8.87.253:46439: connect: Connection refused (error 111),
08:08:09.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:09.670567 9176 heartbeater.cc:461] Registering TS with master...
08:08:09.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:09.670878 10106 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
08:08:09.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:09.674044 10196 ts_manager.cc:194] Registered new tserver with Master: 0b8b4a4be8314be4b54168e19654fe19 (127.8.87.194:43889)
08:08:09.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 23 08:08:09 dist-test-slave-wxqn krb5kdc[8555](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761206888, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.87.193@KRBTEST.COM for kudu/127.8.87.254@KRBTEST.COM
08:08:09.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:08:09.783993 10232 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:08:09.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:08:09.784780 10232 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:08:09.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:08:09.785890 10232 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:08:09.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:09.815867 9029 heartbeater.cc:344] Connected to a master server at 127.8.87.254:43115
08:08:09.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:09.819301 10196 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "b89b7d622ef34a1cb032858329be126a" instance_seqno: 1761206859197829) as {username='kudu', principal='kudu/127.8.87.193@KRBTEST.COM'} at 127.8.87.193:52177; Asking this server to re-register.
08:08:09.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:09.820561 9029 heartbeater.cc:461] Registering TS with master...
08:08:09.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:09.823616 10196 ts_manager.cc:194] Registered new tserver with Master: b89b7d622ef34a1cb032858329be126a (127.8.87.193:36301)
08:08:09.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:08:09.914732 10232 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
08:08:09.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:08:09.915097 10232 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
08:08:09.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:08:09.915632 10232 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:08:09.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:08:09.915989 10232 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
08:08:09.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:08:09.916348 10232 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
08:08:09.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:08:09.916720 10232 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
08:08:09.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:08:09.916992 10232 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:40533/jwks.json
08:08:09.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:09.995146 10232 master_runner.cc:386] Master server non-default flags:
08:08:09.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:08:09.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.87.212:46335
08:08:09.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:08:09.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:08:09.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-1/data
08:08:09.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-1/wal
08:08:09.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
08:08:09.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:08:09.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.87.254:43115,127.8.87.253:46439,127.8.87.252:38507
08:08:09.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/krb5kdc/kudu.keytab
08:08:09.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.8.87.253
08:08:09.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:08:09.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:08:09.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:08:09.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.87.253:46439
08:08:09.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:08:09.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:08:09.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:08:09.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40533/jwks.json
08:08:09.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:08:09.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
08:08:09.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
08:08:09.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:08:09.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:08:09.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-1/data/info.pb
08:08:09.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.87.253
08:08:09.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44589
08:08:09.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:08:09.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:08:09.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:08:09.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:08:09.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:08:09.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-1/logs
08:08:09.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:08:09.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:08:09.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:08:09.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:08:09.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:08:09.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 18b1930e830609248ee2e26d6216d073ff40ac5d
08:08:09.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:08:09.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 23 Oct 2025 07:43:21 UTC on 5fd53c4cbb9d
08:08:09.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8677
08:08:09.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:08:09.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:09.997074 10232 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:08:09.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:09.998991 10232 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:08:10.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:08:10.022336 10261 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:08:10.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 23 08:08:10 dist-test-slave-wxqn krb5kdc[8555](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761206884, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.87.195@KRBTEST.COM for kudu/127.8.87.254@KRBTEST.COM
08:08:10.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:10.129384 9323 heartbeater.cc:344] Connected to a master server at 127.8.87.254:43115
08:08:10.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:10.132375 10196 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "2c485d5a4b9f4ba6926df4b0102eed27" instance_seqno: 1761206864566991) as {username='kudu', principal='kudu/127.8.87.195@KRBTEST.COM'} at 127.8.87.195:59723; Asking this server to re-register.
08:08:10.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:10.133975 9323 heartbeater.cc:461] Registering TS with master...
08:08:10.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:10.137228 10196 ts_manager.cc:194] Registered new tserver with Master: 2c485d5a4b9f4ba6926df4b0102eed27 (127.8.87.195:36087)
08:08:10.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 23 08:08:10 dist-test-slave-wxqn krb5kdc[8555](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761206884, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.87.252@KRBTEST.COM for kudu/127.8.87.254@KRBTEST.COM
08:08:10.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:10.289213 10018 master_runner.cc:186] Error getting master registration for 127.8.87.253:46439: Network error: Client connection negotiation failed: client connection to 127.8.87.253:46439: connect: Connection refused (error 111),
08:08:10.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:10.289706 10018 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
08:08:10.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:10.574930 10247 catalog_manager.cc:1269] Loaded cluster ID: 8f877e344eb84cea8a008e14c9bc53c8
08:08:10.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:10.575328 10247 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 62e54dd7499440acaa1b1c76403da552: loading cluster ID for follower catalog manager: success
08:08:10.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:10.585976 10247 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 62e54dd7499440acaa1b1c76403da552: acquiring CA information for follower catalog manager: success
08:08:10.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:10.602874 10247 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 62e54dd7499440acaa1b1c76403da552: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
08:08:11.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:11.016489 10272 raft_consensus.cc:493] T 00000000000000000000000000000000 P 62e54dd7499440acaa1b1c76403da552 [term 3 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:08:11.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:11.017215 10272 raft_consensus.cc:515] T 00000000000000000000000000000000 P 62e54dd7499440acaa1b1c76403da552 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "62e54dd7499440acaa1b1c76403da552" member_type: VOTER last_known_addr { host: "127.8.87.254" port: 43115 } } peers { permanent_uuid: "4283b8cd94454e13bb77279c6afdb8f2" member_type: VOTER last_known_addr { host: "127.8.87.253" port: 46439 } } peers { permanent_uuid: "b0ae399fb47a454996698251cea1c12d" member_type: VOTER last_known_addr { host: "127.8.87.252" port: 38507 } }
08:08:11.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:08:11.042486 10181 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.87.253:46439: connect: Connection refused (error 111)
08:08:11.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:11.044607 10272 leader_election.cc:290] T 00000000000000000000000000000000 P 62e54dd7499440acaa1b1c76403da552 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 4283b8cd94454e13bb77279c6afdb8f2 (127.8.87.253:46439), b0ae399fb47a454996698251cea1c12d (127.8.87.252:38507)
08:08:11.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:08:11.048185 10181 leader_election.cc:336] T 00000000000000000000000000000000 P 62e54dd7499440acaa1b1c76403da552 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 4283b8cd94454e13bb77279c6afdb8f2 (127.8.87.253:46439): Network error: Client connection negotiation failed: client connection to 127.8.87.253:46439: connect: Connection refused (error 111)
08:08:11.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:11.083665 10079 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "62e54dd7499440acaa1b1c76403da552" candidate_term: 4 candidate_status { last_received { term: 3 index: 6 } } ignore_live_leader: false dest_uuid: "b0ae399fb47a454996698251cea1c12d" is_pre_election: true
08:08:11.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:11.085788 10079 raft_consensus.cc:2468] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d [term 3 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 62e54dd7499440acaa1b1c76403da552 in term 3.
08:08:11.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:11.095059 10180 leader_election.cc:304] T 00000000000000000000000000000000 P 62e54dd7499440acaa1b1c76403da552 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 62e54dd7499440acaa1b1c76403da552, b0ae399fb47a454996698251cea1c12d; no voters: 4283b8cd94454e13bb77279c6afdb8f2
08:08:11.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:11.096398 10272 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 62e54dd7499440acaa1b1c76403da552 [term 3 FOLLOWER]: Leader pre-election won for term 4
08:08:11.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:11.097005 10272 raft_consensus.cc:493] T 00000000000000000000000000000000 P 62e54dd7499440acaa1b1c76403da552 [term 3 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
08:08:11.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:11.097298 10272 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 62e54dd7499440acaa1b1c76403da552 [term 3 FOLLOWER]: Advancing to term 4
08:08:11.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:11.116012 10272 raft_consensus.cc:515] T 00000000000000000000000000000000 P 62e54dd7499440acaa1b1c76403da552 [term 4 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "62e54dd7499440acaa1b1c76403da552" member_type: VOTER last_known_addr { host: "127.8.87.254" port: 43115 } } peers { permanent_uuid: "4283b8cd94454e13bb77279c6afdb8f2" member_type: VOTER last_known_addr { host: "127.8.87.253" port: 46439 } } peers { permanent_uuid: "b0ae399fb47a454996698251cea1c12d" member_type: VOTER last_known_addr { host: "127.8.87.252" port: 38507 } }
08:08:11.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:08:11.125993 10181 leader_election.cc:336] T 00000000000000000000000000000000 P 62e54dd7499440acaa1b1c76403da552 [CANDIDATE]: Term 4 election: RPC error from VoteRequest() call to peer 4283b8cd94454e13bb77279c6afdb8f2 (127.8.87.253:46439): Network error: Client connection negotiation failed: client connection to 127.8.87.253:46439: connect: Connection refused (error 111)
08:08:11.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:11.128965 10272 leader_election.cc:290] T 00000000000000000000000000000000 P 62e54dd7499440acaa1b1c76403da552 [CANDIDATE]: Term 4 election: Requested vote from peers 4283b8cd94454e13bb77279c6afdb8f2 (127.8.87.253:46439), b0ae399fb47a454996698251cea1c12d (127.8.87.252:38507)
08:08:11.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:11.129530 10079 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "62e54dd7499440acaa1b1c76403da552" candidate_term: 4 candidate_status { last_received { term: 3 index: 6 } } ignore_live_leader: false dest_uuid: "b0ae399fb47a454996698251cea1c12d"
08:08:11.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:11.130354 10079 raft_consensus.cc:3060] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d [term 3 FOLLOWER]: Advancing to term 4
08:08:11.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:11.144717 10079 raft_consensus.cc:2468] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d [term 4 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 62e54dd7499440acaa1b1c76403da552 in term 4.
08:08:11.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:11.146054 10180 leader_election.cc:304] T 00000000000000000000000000000000 P 62e54dd7499440acaa1b1c76403da552 [CANDIDATE]: Term 4 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 62e54dd7499440acaa1b1c76403da552, b0ae399fb47a454996698251cea1c12d; no voters: 4283b8cd94454e13bb77279c6afdb8f2
08:08:11.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:11.147564 10272 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 62e54dd7499440acaa1b1c76403da552 [term 4 FOLLOWER]: Leader election won for term 4
08:08:11.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:11.158435 10272 raft_consensus.cc:697] T 00000000000000000000000000000000 P 62e54dd7499440acaa1b1c76403da552 [term 4 LEADER]: Becoming Leader. State: Replica: 62e54dd7499440acaa1b1c76403da552, State: Running, Role: LEADER
08:08:11.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:11.160006 10272 consensus_queue.cc:237] T 00000000000000000000000000000000 P 62e54dd7499440acaa1b1c76403da552 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 3.6, Last appended by leader: 6, Current term: 4, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "62e54dd7499440acaa1b1c76403da552" member_type: VOTER last_known_addr { host: "127.8.87.254" port: 43115 } } peers { permanent_uuid: "4283b8cd94454e13bb77279c6afdb8f2" member_type: VOTER last_known_addr { host: "127.8.87.253" port: 46439 } } peers { permanent_uuid: "b0ae399fb47a454996698251cea1c12d" member_type: VOTER last_known_addr { host: "127.8.87.252" port: 38507 } }
08:08:11.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:11.174677 10278 sys_catalog.cc:455] T 00000000000000000000000000000000 P 62e54dd7499440acaa1b1c76403da552 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 62e54dd7499440acaa1b1c76403da552. Latest consensus state: current_term: 4 leader_uuid: "62e54dd7499440acaa1b1c76403da552" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "62e54dd7499440acaa1b1c76403da552" member_type: VOTER last_known_addr { host: "127.8.87.254" port: 43115 } } peers { permanent_uuid: "4283b8cd94454e13bb77279c6afdb8f2" member_type: VOTER last_known_addr { host: "127.8.87.253" port: 46439 } } peers { permanent_uuid: "b0ae399fb47a454996698251cea1c12d" member_type: VOTER last_known_addr { host: "127.8.87.252" port: 38507 } } }
08:08:11.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:11.175873 10278 sys_catalog.cc:458] T 00000000000000000000000000000000 P 62e54dd7499440acaa1b1c76403da552 [sys.catalog]: This master's current role is: LEADER
08:08:11.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:11.185336 10280 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
08:08:11.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:11.197084 10280 catalog_manager.cc:1494] Initializing Kudu cluster ID...
08:08:11.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:11.203909 10280 catalog_manager.cc:1269] Loaded cluster ID: 8f877e344eb84cea8a008e14c9bc53c8
08:08:11.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:11.204483 10280 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
08:08:11.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:11.210050 10280 catalog_manager.cc:1514] Loading token signing keys...
08:08:11.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:11.215533 10280 catalog_manager.cc:6033] T 00000000000000000000000000000000 P 62e54dd7499440acaa1b1c76403da552: Loaded TSK: 0
08:08:11.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:11.217391 10280 catalog_manager.cc:1524] Initializing in-progress tserver states...
08:08:11.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 23 08:08:11 dist-test-slave-wxqn krb5kdc[8555](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761206891, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.87.194@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:08:11.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:11.744140 9056 init.cc:295] Successfully reacquired a new kerberos TGT
08:08:11.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:11.762354 10079 raft_consensus.cc:1275] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d [term 4 FOLLOWER]: Refusing update from remote peer 62e54dd7499440acaa1b1c76403da552: Log matching property violated. Preceding OpId in replica: term: 3 index: 6. Preceding OpId from leader: term: 4 index: 7. (index mismatch)
08:08:11.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:08:11.768453 10181 consensus_peers.cc:597] T 00000000000000000000000000000000 P 62e54dd7499440acaa1b1c76403da552 -> Peer 4283b8cd94454e13bb77279c6afdb8f2 (127.8.87.253:46439): Couldn't send request to peer 4283b8cd94454e13bb77279c6afdb8f2. Status: Network error: Client connection negotiation failed: client connection to 127.8.87.253:46439: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
08:08:11.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:11.771629 10283 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 62e54dd7499440acaa1b1c76403da552 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b0ae399fb47a454996698251cea1c12d" member_type: VOTER last_known_addr { host: "127.8.87.252" port: 38507 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.001s
08:08:11.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:11.832510 10282 sys_catalog.cc:455] T 00000000000000000000000000000000 P 62e54dd7499440acaa1b1c76403da552 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 4 leader_uuid: "62e54dd7499440acaa1b1c76403da552" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "62e54dd7499440acaa1b1c76403da552" member_type: VOTER last_known_addr { host: "127.8.87.254" port: 43115 } } peers { permanent_uuid: "4283b8cd94454e13bb77279c6afdb8f2" member_type: VOTER last_known_addr { host: "127.8.87.253" port: 46439 } } peers { permanent_uuid: "b0ae399fb47a454996698251cea1c12d" member_type: VOTER last_known_addr { host: "127.8.87.252" port: 38507 } } }
08:08:11.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:11.833288 10282 sys_catalog.cc:458] T 00000000000000000000000000000000 P 62e54dd7499440acaa1b1c76403da552 [sys.catalog]: This master's current role is: LEADER
08:08:11.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:11.843153 10286 sys_catalog.cc:455] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d [sys.catalog]: SysCatalogTable state changed. Reason: New leader 62e54dd7499440acaa1b1c76403da552. Latest consensus state: current_term: 4 leader_uuid: "62e54dd7499440acaa1b1c76403da552" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "62e54dd7499440acaa1b1c76403da552" member_type: VOTER last_known_addr { host: "127.8.87.254" port: 43115 } } peers { permanent_uuid: "4283b8cd94454e13bb77279c6afdb8f2" member_type: VOTER last_known_addr { host: "127.8.87.253" port: 46439 } } peers { permanent_uuid: "b0ae399fb47a454996698251cea1c12d" member_type: VOTER last_known_addr { host: "127.8.87.252" port: 38507 } } }
08:08:11.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:11.843982 10286 sys_catalog.cc:458] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d [sys.catalog]: This master's current role is: FOLLOWER
08:08:11.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:11.862973 10286 sys_catalog.cc:455] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 4 leader_uuid: "62e54dd7499440acaa1b1c76403da552" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "62e54dd7499440acaa1b1c76403da552" member_type: VOTER last_known_addr { host: "127.8.87.254" port: 43115 } } peers { permanent_uuid: "4283b8cd94454e13bb77279c6afdb8f2" member_type: VOTER last_known_addr { host: "127.8.87.253" port: 46439 } } peers { permanent_uuid: "b0ae399fb47a454996698251cea1c12d" member_type: VOTER last_known_addr { host: "127.8.87.252" port: 38507 } } }
08:08:11.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:11.863718 10286 sys_catalog.cc:458] T 00000000000000000000000000000000 P b0ae399fb47a454996698251cea1c12d [sys.catalog]: This master's current role is: FOLLOWER
08:08:12.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:08:11.419539 10260 debug-util.cc:398] Leaking SignalData structure 0x7b0800027080 after lost signal to thread 10232
08:08:12.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:08:10.024412 10262 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:08:12.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:08:12.072141 10232 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 2.050s user 0.443s sys 0.706s
08:08:12.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:08:12.077754 10232 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 2.056s user 0.444s sys 0.707s
08:08:12.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:08:12.093684 10294 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:08:12.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:12.104857 10232 server_base.cc:1047] running on GCE node
08:08:12.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:12.106542 10232 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:08:12.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:12.119093 10232 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:08:12.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:12.121454 10232 hybrid_clock.cc:648] HybridClock initialized: now 1761206892121406 us; error 53 us; skew 500 ppm
08:08:12.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:12.122646 10232 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:08:12.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 23 08:08:12 dist-test-slave-wxqn krb5kdc[8555](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761206892, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.87.253@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:08:12.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:12.131994 10232 init.cc:377] Logged in from keytab as kudu/127.8.87.253@KRBTEST.COM (short username kudu)
08:08:12.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:12.154273 10232 webserver.cc:492] Webserver started at http://127.8.87.253:44589/ using document root <none> and password file <none>
08:08:12.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:12.155529 10232 fs_manager.cc:362] Metadata directory not provided
08:08:12.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:12.155822 10232 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:08:12.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:12.167104 10232 fs_manager.cc:714] Time spent opening directory manager: real 0.007s user 0.004s sys 0.004s
08:08:12.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:12.185055 10302 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:08:12.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:12.186650 10232 fs_manager.cc:730] Time spent opening block manager: real 0.015s user 0.002s sys 0.010s
08:08:12.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:12.187141 10232 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-1/data,/tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-1/wal
08:08:12.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4283b8cd94454e13bb77279c6afdb8f2"
08:08:12.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-23 08:07:31 on dist-test-slave-wxqn"
08:08:12.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:12.191393 10232 fs_report.cc:389] FS layout report
08:08:12.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:08:12.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-1/wal
08:08:12.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-1/wal
08:08:12.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-1/data/data
08:08:12.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:08:12.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:08:12.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:08:12.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:08:12.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:08:12.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:08:12.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:08:12.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:08:12.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:08:12.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:08:12.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:08:12.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:12.315663 10232 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:08:12.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:12.321717 10232 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:08:12.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:12.322218 10232 kserver.cc:163] Server-wide thread pool size limit: 3276
08:08:12.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:12.454641 10232 rpc_server.cc:307] RPC server started. Bound to: 127.8.87.253:46439
08:08:12.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:12.454717 10353 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.8.87.253:46439 every 8 connection(s)
08:08:12.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:12.459543 10232 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskcrgkbR/test-tmp/mini-kudu-cluster9154750853370229683/master-1/data/info.pb
08:08:12.461 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:08:12.462 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.8.87.252:38507(127.8.87.252:38507)] connecting to peer
08:08:12.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:12.460707 8543 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskcrgkbR/build/tsan/bin/kudu as pid 10232
08:08:12.466 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.8.87.252:38507(127.8.87.252:38507)] Successfully connected to peer
08:08:12.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:12.479388 10354 sys_catalog.cc:263] Verifying existing consensus state
08:08:12.484 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.87.252:38507(127.8.87.252:38507)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
08:08:12.486 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.8.87.252:38507(127.8.87.252:38507)] handling channelInactive
08:08:12.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 08:08:12.489367 10355 negotiation.cc:337] Failed RPC negotiation. Trace:
08:08:12.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1023 08:08:12.466016 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:37822 (local address 127.8.87.252:38507)
08:08:12.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1023 08:08:12.481960 (+ 15944us) server_negotiation.cc:207] Beginning negotiation
08:08:12.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1023 08:08:12.482014 (+ 54us) server_negotiation.cc:400] Waiting for connection header
08:08:12.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1023 08:08:12.482094 (+ 80us) server_negotiation.cc:408] Connection header received
08:08:12.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1023 08:08:12.482302 (+ 208us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
08:08:12.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1023 08:08:12.482344 (+ 42us) server_negotiation.cc:462] Received NEGOTIATE request from client
08:08:12.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1023 08:08:12.482664 (+ 320us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
08:08:12.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1023 08:08:12.486383 (+ 3719us) server_negotiation.cc:227] Negotiated authn=SASL
08:08:12.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1023 08:08:12.487367 (+ 984us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:37822: BlockingRecv error: recv got EOF from 127.0.0.1:37822 (error 108)
08:08:12.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":15072,"thread_start_us":14719,"threads_started":1}
08:08:12.490 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.8.87.254:43115(127.8.87.254:43115)] disconnecting while in state TERMINATED
08:08:12.496 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.8.87.253:46439(127.8.87.253:46439)] disconnecting while in state TERMINATED
08:08:12.497 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.8.87.252:38507(127.8.87.252:38507)] disconnecting while in state TERMINATED
08:08:12.509 [DEBUG - Time-limited test] (AsyncKuduClient.java:2704) Releasing all remaining resources
08:08:12.521 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
08:08:12.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:12.526528 10354 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4283b8cd94454e13bb77279c6afdb8f2: Bootstrap starting.
08:08:12.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:12.542299 8543 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskcrgkbR/build/tsan/bin/kudu with pid 8887
08:08:12.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:12.585044 8543 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskcrgkbR/build/tsan/bin/kudu with pid 9039
08:08:12.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:12.610790 10354 log.cc:826] T 00000000000000000000000000000000 P 4283b8cd94454e13bb77279c6afdb8f2: Log is configured to *not* fsync() on all Append() calls
08:08:12.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:12.627727 8543 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskcrgkbR/build/tsan/bin/kudu with pid 9188
08:08:12.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:12.654207 10354 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4283b8cd94454e13bb77279c6afdb8f2: Bootstrap replayed 1/1 log segments. Stats: ops{read=6 overwritten=0 applied=6 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
08:08:12.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:12.655328 10354 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4283b8cd94454e13bb77279c6afdb8f2: Bootstrap complete.
08:08:12.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:12.659890 8543 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskcrgkbR/build/tsan/bin/kudu with pid 10106
08:08:12.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:12.700057 8543 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskcrgkbR/build/tsan/bin/kudu with pid 10232
08:08:12.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:12.719041 8543 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskcrgkbR/build/tsan/bin/kudu with pid 10018
08:08:13.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-10-23T08:08:13Z chronyd exiting
08:08:14.991 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
08:08:14.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:14.992908 7815 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskcrgkbR/build/tsan/bin/kudu with pid 8097
08:08:15.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:15.020766 7815 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskcrgkbR/build/tsan/bin/kudu with pid 8263
08:08:15.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:15.049247 7815 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskcrgkbR/build/tsan/bin/kudu with pid 8401
08:08:15.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:15.078258 7815 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskcrgkbR/build/tsan/bin/kudu with pid 7827
08:08:15.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:15.112131 7815 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskcrgkbR/build/tsan/bin/kudu with pid 7897
08:08:15.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 08:08:15.143785 7815 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskcrgkbR/build/tsan/bin/kudu with pid 7972
08:08:16.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-10-23T08:08:16Z chronyd exiting
08:08:17.313 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.Negotiator.chooseAndInitializeSaslMech(Negotiator.java:565) ~[main/:?]
at org.apache.kudu.client.Negotiator.handleNegotiateResponse(Negotiator.java:433) ~[main/:?]
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:367) ~[main/:?]
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:94) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:748)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.Negotiator.chooseAndInitializeSaslMech(Negotiator.java:565)
at org.apache.kudu.client.Negotiator.handleNegotiateResponse(Negotiator.java:433)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:367)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:94)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more