02:05:57.862 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
02:05:57.874 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-task8JcgIg/test-tmp
02:05:57.877 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
02:05:57.880 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-task8JcgIg/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
02:05:58.127 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster5757810492041090520"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
02:05:58.277 [DEBUG - main] (MiniKuduCluster.java:195) Response:
02:05:58.278 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
02:05:58.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:05:58.281659 7379 mini_oidc.cc:150] Starting JWKS server
02:05:58.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:05:58.285626 7379 webserver.cc:469] Webserver started at http://127.0.0.1:38165/ using document root <none> and password file <none>
02:05:58.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:05:58.286052 7379 mini_oidc.cc:186] Starting OIDC Discovery server
02:05:58.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:05:58.287909 7379 webserver.cc:469] Webserver started at http://127.0.0.1:37385/ using document root <none> and password file <none>
02:05:58.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-06-28T02:05:58Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
02:05:58.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-06-28T02:05:58Z Disabled control of system clock
02:05:58.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:05:58.350736 7379 external_mini_cluster.cc:1351] Running /tmp/dist-test-task8JcgIg/build/tsan/bin/kudu
02:05:58.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task8JcgIg/build/tsan/bin/kudu
02:05:58.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster5757810492041090520/master-0/wal
02:05:58.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster5757810492041090520/master-0/data
02:05:58.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:05:58.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:05:58.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:05:58.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:05:58.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:05:58.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:05:58.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster5757810492041090520/master-0/logs
02:05:58.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster5757810492041090520/master-0/data/info.pb
02:05:58.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:05:58.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:05:58.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:05:58.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:05:58.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:05:58.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:05:58.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:05:58.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:05:58.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:05:58.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:05:58.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:05:58.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:05:58.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.52.254:46311
02:05:58.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.52.254
02:05:58.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:05:58.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.52.212:44749
02:05:58.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:05:58.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:05:58.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:05:58.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:05:58.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.52.254:46311,127.7.52.253:41921,127.7.52.252:43529
02:05:58.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:05:58.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38165/jwks.json with env {}
02:05:58.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:05:58.646076 7391 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:05:58.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:05:58.646621 7391 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:05:58.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:05:58.647189 7391 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:05:58.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:05:58.678342 7391 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
02:05:58.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:05:58.678637 7391 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:05:58.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:05:58.678848 7391 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
02:05:58.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:05:58.679095 7391 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
02:05:58.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:05:58.679330 7391 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
02:05:58.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:05:58.679473 7391 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38165/jwks.json
02:05:58.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:05:58.716688 7391 master_runner.cc:386] Master server non-default flags:
02:05:58.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:05:58.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.52.212:44749
02:05:58.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:05:58.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:05:58.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster5757810492041090520/master-0/data
02:05:58.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster5757810492041090520/master-0/wal
02:05:58.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:05:58.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.52.254:46311,127.7.52.253:41921,127.7.52.252:43529
02:05:58.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:05:58.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:05:58.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:05:58.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.52.254:46311
02:05:58.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:05:58.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:05:58.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:05:58.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38165/jwks.json
02:05:58.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:05:58.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:05:58.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster5757810492041090520/master-0/data/info.pb
02:05:58.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.52.254
02:05:58.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:05:58.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:05:58.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:05:58.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:05:58.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:05:58.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:05:58.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster5757810492041090520/master-0/logs
02:05:58.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:05:58.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:05:58.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:05:58.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:05:58.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
02:05:58.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision f0776864a65c19ebdd909053f3e624e8bcc25cee
02:05:58.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:05:58.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 28 Jun 2025 01:43:19 UTC on 5fd53c4cbb9d
02:05:58.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 6805
02:05:58.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:05:58.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:05:58.718232 7391 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:05:58.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:05:58.720572 7391 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:05:58.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:05:58.738279 7398 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:05:59.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:05:58.738577 7397 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:05:59.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:05:58.738763 7400 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:05:59.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:05:59.927718 7399 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
02:05:59.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:05:59.927831 7391 server_base.cc:1043] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:05:59.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:05:59.933310 7391 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:05:59.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:05:59.936187 7391 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:05:59.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:05:59.937558 7391 hybrid_clock.cc:648] HybridClock initialized: now 1751076359937504 us; error 70 us; skew 500 ppm
02:05:59.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:05:59.939055 7391 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:05:59.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:05:59.950716 7391 webserver.cc:469] Webserver started at http://127.7.52.254:33099/ using document root <none> and password file <none>
02:05:59.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:05:59.951804 7391 fs_manager.cc:362] Metadata directory not provided
02:05:59.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:05:59.952035 7391 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:05:59.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:05:59.952637 7391 server_base.cc:896] This appears to be a new deployment of Kudu; creating new FS layout
02:05:59.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:05:59.957134 7391 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster5757810492041090520/master-0/data/instance:
02:05:59.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "98276328da3f46a28846576a2d5ab83e"
02:05:59.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-28 02:05:59 on dist-test-slave-lq1z"
02:05:59.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:05:59.958453 7391 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster5757810492041090520/master-0/wal/instance:
02:05:59.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "98276328da3f46a28846576a2d5ab83e"
02:05:59.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-28 02:05:59 on dist-test-slave-lq1z"
02:05:59.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:05:59.965591 7391 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.002s sys 0.006s
02:05:59.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:05:59.971216 7407 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:05:59.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:05:59.972443 7391 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
02:05:59.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:05:59.972813 7391 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster5757810492041090520/master-0/data,/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster5757810492041090520/master-0/wal
02:05:59.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "98276328da3f46a28846576a2d5ab83e"
02:05:59.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-28 02:05:59 on dist-test-slave-lq1z"
02:05:59.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:05:59.973212 7391 fs_report.cc:389] FS layout report
02:05:59.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:05:59.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster5757810492041090520/master-0/wal
02:05:59.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster5757810492041090520/master-0/wal
02:05:59.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster5757810492041090520/master-0/data/data
02:05:59.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:05:59.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:05:59.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:05:59.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:05:59.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:05:59.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:05:59.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:05:59.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:05:59.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:05:59.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:05:59.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:06:00.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:00.043344 7391 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:06:00.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:00.045044 7391 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:06:00.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:00.045564 7391 kserver.cc:163] Server-wide thread pool size limit: 3276
02:06:00.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:00.116921 7391 rpc_server.cc:307] RPC server started. Bound to: 127.7.52.254:46311
02:06:00.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:00.117003 7458 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.7.52.254:46311 every 8 connection(s)
02:06:00.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:00.119805 7391 server_base.cc:1180] Dumped server information to /tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster5757810492041090520/master-0/data/info.pb
02:06:00.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:00.125033 7459 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
02:06:00.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:00.127099 7379 external_mini_cluster.cc:1413] Started /tmp/dist-test-task8JcgIg/build/tsan/bin/kudu as pid 7391
02:06:00.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:00.127560 7379 external_mini_cluster.cc:1427] Reading /tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster5757810492041090520/master-0/wal/instance
02:06:00.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:00.131032 7379 external_mini_cluster.cc:1351] Running /tmp/dist-test-task8JcgIg/build/tsan/bin/kudu
02:06:00.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task8JcgIg/build/tsan/bin/kudu
02:06:00.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster5757810492041090520/master-1/wal
02:06:00.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster5757810492041090520/master-1/data
02:06:00.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:06:00.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:06:00.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:06:00.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:06:00.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:06:00.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:06:00.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster5757810492041090520/master-1/logs
02:06:00.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster5757810492041090520/master-1/data/info.pb
02:06:00.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:06:00.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:06:00.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:06:00.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:06:00.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:06:00.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:06:00.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:06:00.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:06:00.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:06:00.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:06:00.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:06:00.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:06:00.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.52.253:41921
02:06:00.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.52.253
02:06:00.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:06:00.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.52.212:44749
02:06:00.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:06:00.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:06:00.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:06:00.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:06:00.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.52.254:46311,127.7.52.253:41921,127.7.52.252:43529
02:06:00.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:06:00.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38165/jwks.json with env {}
02:06:00.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:00.137104 7459 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.52.254" port: 46311 } has no permanent_uuid. Determining permanent_uuid...
02:06:00.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:00.164515 7459 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.52.253" port: 41921 } has no permanent_uuid. Determining permanent_uuid...
02:06:00.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:00.166920 7410 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.52.253:41921: connect: Connection refused (error 111)
02:06:00.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:00.170974 7459 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.7.52.253:41921: Network error: Client connection negotiation failed: client connection to 127.7.52.253:41921: connect: Connection refused (error 111)
02:06:00.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:00.220602 7459 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.52.253" port: 41921 } attempt: 1
02:06:00.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:00.224764 7459 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.7.52.253:41921: Network error: Client connection negotiation failed: client connection to 127.7.52.253:41921: connect: Connection refused (error 111)
02:06:00.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:00.293246 7459 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.52.253" port: 41921 } attempt: 2
02:06:00.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:00.297551 7459 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.7.52.253:41921: Network error: Client connection negotiation failed: client connection to 127.7.52.253:41921: connect: Connection refused (error 111)
02:06:00.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:00.388923 7459 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.52.253" port: 41921 } attempt: 3
02:06:00.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:00.393114 7459 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.7.52.253:41921: Network error: Client connection negotiation failed: client connection to 127.7.52.253:41921: connect: Connection refused (error 111)
02:06:00.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:00.439092 7461 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:06:00.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:00.439678 7461 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:06:00.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:00.440235 7461 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:06:00.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:00.471474 7461 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
02:06:00.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:00.471762 7461 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:06:00.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:00.471956 7461 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
02:06:00.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:00.472195 7461 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
02:06:00.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:00.472384 7461 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
02:06:00.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:00.472540 7461 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38165/jwks.json
02:06:00.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:00.507694 7461 master_runner.cc:386] Master server non-default flags:
02:06:00.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:06:00.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.52.212:44749
02:06:00.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:06:00.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:06:00.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster5757810492041090520/master-1/data
02:06:00.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster5757810492041090520/master-1/wal
02:06:00.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:06:00.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.52.254:46311,127.7.52.253:41921,127.7.52.252:43529
02:06:00.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:06:00.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:06:00.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:06:00.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.52.253:41921
02:06:00.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:06:00.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:06:00.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:06:00.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38165/jwks.json
02:06:00.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:06:00.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:06:00.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster5757810492041090520/master-1/data/info.pb
02:06:00.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.52.253
02:06:00.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:06:00.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:06:00.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:06:00.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:06:00.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:06:00.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:06:00.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster5757810492041090520/master-1/logs
02:06:00.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:06:00.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:06:00.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:06:00.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:06:00.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
02:06:00.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision f0776864a65c19ebdd909053f3e624e8bcc25cee
02:06:00.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:06:00.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 28 Jun 2025 01:43:19 UTC on 5fd53c4cbb9d
02:06:00.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 6805
02:06:00.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:06:00.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:00.509083 7461 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:06:00.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:00.510761 7461 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:06:00.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:00.525835 7471 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:00.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:00.536566 7459 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.52.253" port: 41921 } attempt: 4
02:06:00.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:00.540513 7459 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.7.52.253:41921: Network error: Client connection negotiation failed: client connection to 127.7.52.253:41921: connect: Connection refused (error 111)
02:06:00.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:00.840024 7459 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.52.253" port: 41921 } attempt: 5
02:06:00.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:00.845865 7459 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.7.52.253:41921: Network error: Client connection negotiation failed: client connection to 127.7.52.253:41921: connect: Connection refused (error 111)
02:06:01.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:01.393272 7459 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.52.253" port: 41921 } attempt: 6
02:06:01.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:01.401575 7459 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.7.52.253:41921: Network error: Client connection negotiation failed: client connection to 127.7.52.253:41921: connect: Connection refused (error 111)
02:06:01.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:00.525873 7473 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:01.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:00.527107 7470 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:01.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:00.528260 7461 server_base.cc:1048] running on GCE node
02:06:01.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:01.707643 7461 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:06:01.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:01.710403 7461 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:06:01.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:01.711774 7461 hybrid_clock.cc:648] HybridClock initialized: now 1751076361711726 us; error 61 us; skew 500 ppm
02:06:01.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:01.712908 7461 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:06:01.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:01.719893 7461 webserver.cc:469] Webserver started at http://127.7.52.253:34623/ using document root <none> and password file <none>
02:06:01.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:01.720818 7461 fs_manager.cc:362] Metadata directory not provided
02:06:01.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:01.721002 7461 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:06:01.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:01.721390 7461 server_base.cc:896] This appears to be a new deployment of Kudu; creating new FS layout
02:06:01.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:01.726092 7461 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster5757810492041090520/master-1/data/instance:
02:06:01.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "dbbf338bb3b94a36aa32a3103260e759"
02:06:01.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-28 02:06:01 on dist-test-slave-lq1z"
02:06:01.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:01.727347 7461 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster5757810492041090520/master-1/wal/instance:
02:06:01.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "dbbf338bb3b94a36aa32a3103260e759"
02:06:01.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-28 02:06:01 on dist-test-slave-lq1z"
02:06:01.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:01.734512 7461 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.005s sys 0.000s
02:06:01.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:01.739966 7482 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:06:01.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:01.741153 7461 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
02:06:01.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:01.741441 7461 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster5757810492041090520/master-1/data,/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster5757810492041090520/master-1/wal
02:06:01.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "dbbf338bb3b94a36aa32a3103260e759"
02:06:01.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-28 02:06:01 on dist-test-slave-lq1z"
02:06:01.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:01.741747 7461 fs_report.cc:389] FS layout report
02:06:01.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:06:01.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster5757810492041090520/master-1/wal
02:06:01.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster5757810492041090520/master-1/wal
02:06:01.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster5757810492041090520/master-1/data/data
02:06:01.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:06:01.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:06:01.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:06:01.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:06:01.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:06:01.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:06:01.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:06:01.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:06:01.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:06:01.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:06:01.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:06:01.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:01.825160 7461 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:06:01.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:01.826624 7461 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:06:01.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:01.827031 7461 kserver.cc:163] Server-wide thread pool size limit: 3276
02:06:01.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:01.896754 7461 rpc_server.cc:307] RPC server started. Bound to: 127.7.52.253:41921
02:06:01.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:01.896833 7533 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.7.52.253:41921 every 8 connection(s)
02:06:01.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:01.899348 7461 server_base.cc:1180] Dumped server information to /tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster5757810492041090520/master-1/data/info.pb
02:06:01.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:01.904786 7534 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
02:06:01.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:01.904932 7379 external_mini_cluster.cc:1413] Started /tmp/dist-test-task8JcgIg/build/tsan/bin/kudu as pid 7461
02:06:01.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:01.905257 7379 external_mini_cluster.cc:1427] Reading /tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster5757810492041090520/master-1/wal/instance
02:06:01.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:01.907204 7379 external_mini_cluster.cc:1351] Running /tmp/dist-test-task8JcgIg/build/tsan/bin/kudu
02:06:01.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task8JcgIg/build/tsan/bin/kudu
02:06:01.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster5757810492041090520/master-2/wal
02:06:01.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster5757810492041090520/master-2/data
02:06:01.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:06:01.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:06:01.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:06:01.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:06:01.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:06:01.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:06:01.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster5757810492041090520/master-2/logs
02:06:01.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster5757810492041090520/master-2/data/info.pb
02:06:01.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:06:01.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:06:01.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:06:01.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:06:01.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:06:01.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:06:01.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:06:01.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:06:01.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:06:01.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:06:01.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:06:01.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:06:01.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.52.252:43529
02:06:01.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.52.252
02:06:01.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:06:01.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.52.212:44749
02:06:01.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:06:01.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:06:01.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:06:01.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:06:01.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.52.254:46311,127.7.52.253:41921,127.7.52.252:43529
02:06:01.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:06:01.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38165/jwks.json with env {}
02:06:01.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:01.915812 7534 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.52.254" port: 46311 } has no permanent_uuid. Determining permanent_uuid...
02:06:01.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:01.937485 7534 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.52.253" port: 41921 } has no permanent_uuid. Determining permanent_uuid...
02:06:01.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:01.951936 7534 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.52.252" port: 43529 } has no permanent_uuid. Determining permanent_uuid...
02:06:01.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:01.954412 7484 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.52.252:43529: connect: Connection refused (error 111)
02:06:01.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:01.959790 7534 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.7.52.252:43529: Network error: Client connection negotiation failed: client connection to 127.7.52.252:43529: connect: Connection refused (error 111)
02:06:02.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:02.009452 7534 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.52.252" port: 43529 } attempt: 1
02:06:02.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:02.013258 7534 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.7.52.252:43529: Network error: Client connection negotiation failed: client connection to 127.7.52.252:43529: connect: Connection refused (error 111)
02:06:02.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:02.081751 7534 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.52.252" port: 43529 } attempt: 2
02:06:02.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:02.085566 7534 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.7.52.252:43529: Network error: Client connection negotiation failed: client connection to 127.7.52.252:43529: connect: Connection refused (error 111)
02:06:02.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:02.177026 7534 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.52.252" port: 43529 } attempt: 3
02:06:02.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:02.180847 7534 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.7.52.252:43529: Network error: Client connection negotiation failed: client connection to 127.7.52.252:43529: connect: Connection refused (error 111)
02:06:02.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:02.223501 7536 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:06:02.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:02.224102 7536 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:06:02.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:02.224581 7536 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:06:02.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:02.255196 7536 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
02:06:02.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:02.255486 7536 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:06:02.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:02.255681 7536 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
02:06:02.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:02.255863 7536 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
02:06:02.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:02.256089 7536 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
02:06:02.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:02.256278 7536 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38165/jwks.json
02:06:02.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:02.295734 7536 master_runner.cc:386] Master server non-default flags:
02:06:02.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:06:02.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.52.212:44749
02:06:02.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:06:02.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:06:02.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster5757810492041090520/master-2/data
02:06:02.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster5757810492041090520/master-2/wal
02:06:02.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:06:02.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.52.254:46311,127.7.52.253:41921,127.7.52.252:43529
02:06:02.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:06:02.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:06:02.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:06:02.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.52.252:43529
02:06:02.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:06:02.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:06:02.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:06:02.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38165/jwks.json
02:06:02.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:06:02.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:06:02.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster5757810492041090520/master-2/data/info.pb
02:06:02.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.52.252
02:06:02.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:06:02.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:06:02.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:06:02.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:06:02.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:06:02.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:06:02.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster5757810492041090520/master-2/logs
02:06:02.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:06:02.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:06:02.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:06:02.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:06:02.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
02:06:02.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision f0776864a65c19ebdd909053f3e624e8bcc25cee
02:06:02.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:06:02.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 28 Jun 2025 01:43:19 UTC on 5fd53c4cbb9d
02:06:02.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 6805
02:06:02.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:06:02.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:02.297417 7536 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:06:02.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:02.299715 7536 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:06:02.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:02.314847 7546 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:02.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:02.324208 7534 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.52.252" port: 43529 } attempt: 4
02:06:02.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:02.327838 7534 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.7.52.252:43529: Network error: Client connection negotiation failed: client connection to 127.7.52.252:43529: connect: Connection refused (error 111)
02:06:02.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:02.462065 7459 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.52.253" port: 41921 } attempt: 7
02:06:02.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:02.474004 7459 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.52.252" port: 43529 } has no permanent_uuid. Determining permanent_uuid...
02:06:02.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:02.481709 7459 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.7.52.252:43529: Network error: Client connection negotiation failed: client connection to 127.7.52.252:43529: connect: Connection refused (error 111)
02:06:02.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:02.540374 7459 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.52.252" port: 43529 } attempt: 1
02:06:02.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:02.546132 7459 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.7.52.252:43529: Network error: Client connection negotiation failed: client connection to 127.7.52.252:43529: connect: Connection refused (error 111)
02:06:02.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:02.627358 7534 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.52.252" port: 43529 } attempt: 5
02:06:02.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:02.627590 7459 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.52.252" port: 43529 } attempt: 2
02:06:02.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:02.631785 7459 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.7.52.252:43529: Network error: Client connection negotiation failed: client connection to 127.7.52.252:43529: connect: Connection refused (error 111)
02:06:02.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:02.632949 7534 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.7.52.252:43529: Network error: Client connection negotiation failed: client connection to 127.7.52.252:43529: connect: Connection refused (error 111)
02:06:02.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:02.717247 7459 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.52.252" port: 43529 } attempt: 3
02:06:02.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:02.723439 7459 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.7.52.252:43529: Network error: Client connection negotiation failed: client connection to 127.7.52.252:43529: connect: Connection refused (error 111)
02:06:02.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:02.864127 7459 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.52.252" port: 43529 } attempt: 4
02:06:02.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:02.870060 7459 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.7.52.252:43529: Network error: Client connection negotiation failed: client connection to 127.7.52.252:43529: connect: Connection refused (error 111)
02:06:03.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:03.153472 7459 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.52.252" port: 43529 } attempt: 5
02:06:03.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:03.157723 7459 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.7.52.252:43529: Network error: Client connection negotiation failed: client connection to 127.7.52.252:43529: connect: Connection refused (error 111)
02:06:03.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:03.180318 7534 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.52.252" port: 43529 } attempt: 6
02:06:03.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:03.188115 7534 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.7.52.252:43529: Network error: Client connection negotiation failed: client connection to 127.7.52.252:43529: connect: Connection refused (error 111)
02:06:03.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:02.315104 7547 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:03.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:02.317076 7536 server_base.cc:1048] running on GCE node
02:06:03.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:02.315385 7549 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:03.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:03.503458 7536 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:06:03.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:03.506577 7536 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:06:03.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:03.508008 7536 hybrid_clock.cc:648] HybridClock initialized: now 1751076363507955 us; error 73 us; skew 500 ppm
02:06:03.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:03.508869 7536 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:06:03.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:03.515677 7536 webserver.cc:469] Webserver started at http://127.7.52.252:46085/ using document root <none> and password file <none>
02:06:03.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:03.516609 7536 fs_manager.cc:362] Metadata directory not provided
02:06:03.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:03.516803 7536 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:06:03.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:03.517192 7536 server_base.cc:896] This appears to be a new deployment of Kudu; creating new FS layout
02:06:03.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:03.523782 7536 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster5757810492041090520/master-2/data/instance:
02:06:03.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1860f38db9cc4b6bb85fa7f2d490f6b2"
02:06:03.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-28 02:06:03 on dist-test-slave-lq1z"
02:06:03.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:03.525411 7536 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster5757810492041090520/master-2/wal/instance:
02:06:03.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1860f38db9cc4b6bb85fa7f2d490f6b2"
02:06:03.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-28 02:06:03 on dist-test-slave-lq1z"
02:06:03.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:03.534886 7536 fs_manager.cc:696] Time spent creating directory manager: real 0.009s user 0.009s sys 0.000s
02:06:03.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:03.541925 7561 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:06:03.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:03.543107 7536 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.003s
02:06:03.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:03.543404 7536 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster5757810492041090520/master-2/data,/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster5757810492041090520/master-2/wal
02:06:03.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1860f38db9cc4b6bb85fa7f2d490f6b2"
02:06:03.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-28 02:06:03 on dist-test-slave-lq1z"
02:06:03.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:03.543720 7536 fs_report.cc:389] FS layout report
02:06:03.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:06:03.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster5757810492041090520/master-2/wal
02:06:03.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster5757810492041090520/master-2/wal
02:06:03.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster5757810492041090520/master-2/data/data
02:06:03.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:06:03.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:06:03.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:06:03.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:06:03.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:06:03.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:06:03.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:06:03.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:06:03.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:06:03.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:06:03.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:06:03.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:03.600939 7536 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:06:03.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:03.602653 7536 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:06:03.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:03.603067 7536 kserver.cc:163] Server-wide thread pool size limit: 3276
02:06:03.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:03.674896 7536 rpc_server.cc:307] RPC server started. Bound to: 127.7.52.252:43529
02:06:03.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:03.674978 7612 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.7.52.252:43529 every 8 connection(s)
02:06:03.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:03.677703 7536 server_base.cc:1180] Dumped server information to /tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster5757810492041090520/master-2/data/info.pb
02:06:03.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:03.682664 7613 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
02:06:03.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:03.685631 7379 external_mini_cluster.cc:1413] Started /tmp/dist-test-task8JcgIg/build/tsan/bin/kudu as pid 7536
02:06:03.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:03.686038 7379 external_mini_cluster.cc:1427] Reading /tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster5757810492041090520/master-2/wal/instance
02:06:03.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:03.694350 7613 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.52.254" port: 46311 } has no permanent_uuid. Determining permanent_uuid...
02:06:03.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:03.710362 7459 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.52.252" port: 43529 } attempt: 6
02:06:03.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:03.723142 7613 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.52.253" port: 41921 } has no permanent_uuid. Determining permanent_uuid...
02:06:03.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:03.736408 7613 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.52.252" port: 43529 } has no permanent_uuid. Determining permanent_uuid...
02:06:03.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:03.747238 7459 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 98276328da3f46a28846576a2d5ab83e: Bootstrap starting.
02:06:03.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:03.754838 7459 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 98276328da3f46a28846576a2d5ab83e: Neither blocks nor log segments found. Creating new log.
02:06:03.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:03.757208 7613 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1860f38db9cc4b6bb85fa7f2d490f6b2: Bootstrap starting.
02:06:03.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:03.757639 7459 log.cc:826] T 00000000000000000000000000000000 P 98276328da3f46a28846576a2d5ab83e: Log is configured to *not* fsync() on all Append() calls
02:06:03.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:03.766853 7613 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 1860f38db9cc4b6bb85fa7f2d490f6b2: Neither blocks nor log segments found. Creating new log.
02:06:03.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:03.768702 7613 log.cc:826] T 00000000000000000000000000000000 P 1860f38db9cc4b6bb85fa7f2d490f6b2: Log is configured to *not* fsync() on all Append() calls
02:06:03.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:03.769496 7459 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 98276328da3f46a28846576a2d5ab83e: No bootstrap required, opened a new log
02:06:03.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:03.774125 7613 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1860f38db9cc4b6bb85fa7f2d490f6b2: No bootstrap required, opened a new log
02:06:03.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:03.792677 7459 raft_consensus.cc:357] T 00000000000000000000000000000000 P 98276328da3f46a28846576a2d5ab83e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "98276328da3f46a28846576a2d5ab83e" member_type: VOTER last_known_addr { host: "127.7.52.254" port: 46311 } } peers { permanent_uuid: "dbbf338bb3b94a36aa32a3103260e759" member_type: VOTER last_known_addr { host: "127.7.52.253" port: 41921 } } peers { permanent_uuid: "1860f38db9cc4b6bb85fa7f2d490f6b2" member_type: VOTER last_known_addr { host: "127.7.52.252" port: 43529 } }
02:06:03.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:03.793320 7459 raft_consensus.cc:383] T 00000000000000000000000000000000 P 98276328da3f46a28846576a2d5ab83e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:06:03.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:03.793601 7459 raft_consensus.cc:738] T 00000000000000000000000000000000 P 98276328da3f46a28846576a2d5ab83e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 98276328da3f46a28846576a2d5ab83e, State: Initialized, Role: FOLLOWER
02:06:03.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:03.793218 7613 raft_consensus.cc:357] T 00000000000000000000000000000000 P 1860f38db9cc4b6bb85fa7f2d490f6b2 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "98276328da3f46a28846576a2d5ab83e" member_type: VOTER last_known_addr { host: "127.7.52.254" port: 46311 } } peers { permanent_uuid: "dbbf338bb3b94a36aa32a3103260e759" member_type: VOTER last_known_addr { host: "127.7.52.253" port: 41921 } } peers { permanent_uuid: "1860f38db9cc4b6bb85fa7f2d490f6b2" member_type: VOTER last_known_addr { host: "127.7.52.252" port: 43529 } }
02:06:03.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:03.794051 7613 raft_consensus.cc:383] T 00000000000000000000000000000000 P 1860f38db9cc4b6bb85fa7f2d490f6b2 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:06:03.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:03.794377 7613 raft_consensus.cc:738] T 00000000000000000000000000000000 P 1860f38db9cc4b6bb85fa7f2d490f6b2 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1860f38db9cc4b6bb85fa7f2d490f6b2, State: Initialized, Role: FOLLOWER
02:06:03.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:03.794648 7459 consensus_queue.cc:260] T 00000000000000000000000000000000 P 98276328da3f46a28846576a2d5ab83e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "98276328da3f46a28846576a2d5ab83e" member_type: VOTER last_known_addr { host: "127.7.52.254" port: 46311 } } peers { permanent_uuid: "dbbf338bb3b94a36aa32a3103260e759" member_type: VOTER last_known_addr { host: "127.7.52.253" port: 41921 } } peers { permanent_uuid: "1860f38db9cc4b6bb85fa7f2d490f6b2" member_type: VOTER last_known_addr { host: "127.7.52.252" port: 43529 } }
02:06:03.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:03.795260 7613 consensus_queue.cc:260] T 00000000000000000000000000000000 P 1860f38db9cc4b6bb85fa7f2d490f6b2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "98276328da3f46a28846576a2d5ab83e" member_type: VOTER last_known_addr { host: "127.7.52.254" port: 46311 } } peers { permanent_uuid: "dbbf338bb3b94a36aa32a3103260e759" member_type: VOTER last_known_addr { host: "127.7.52.253" port: 41921 } } peers { permanent_uuid: "1860f38db9cc4b6bb85fa7f2d490f6b2" member_type: VOTER last_known_addr { host: "127.7.52.252" port: 43529 } }
02:06:03.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:03.799671 7625 sys_catalog.cc:455] T 00000000000000000000000000000000 P 98276328da3f46a28846576a2d5ab83e [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "98276328da3f46a28846576a2d5ab83e" member_type: VOTER last_known_addr { host: "127.7.52.254" port: 46311 } } peers { permanent_uuid: "dbbf338bb3b94a36aa32a3103260e759" member_type: VOTER last_known_addr { host: "127.7.52.253" port: 41921 } } peers { permanent_uuid: "1860f38db9cc4b6bb85fa7f2d490f6b2" member_type: VOTER last_known_addr { host: "127.7.52.252" port: 43529 } } }
02:06:03.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:03.800792 7625 sys_catalog.cc:458] T 00000000000000000000000000000000 P 98276328da3f46a28846576a2d5ab83e [sys.catalog]: This master's current role is: FOLLOWER
02:06:03.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:03.801837 7459 sys_catalog.cc:564] T 00000000000000000000000000000000 P 98276328da3f46a28846576a2d5ab83e [sys.catalog]: configured and running, proceeding with master startup.
02:06:03.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:03.802392 7626 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1860f38db9cc4b6bb85fa7f2d490f6b2 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "98276328da3f46a28846576a2d5ab83e" member_type: VOTER last_known_addr { host: "127.7.52.254" port: 46311 } } peers { permanent_uuid: "dbbf338bb3b94a36aa32a3103260e759" member_type: VOTER last_known_addr { host: "127.7.52.253" port: 41921 } } peers { permanent_uuid: "1860f38db9cc4b6bb85fa7f2d490f6b2" member_type: VOTER last_known_addr { host: "127.7.52.252" port: 43529 } } }
02:06:03.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:03.803215 7626 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1860f38db9cc4b6bb85fa7f2d490f6b2 [sys.catalog]: This master's current role is: FOLLOWER
02:06:03.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:03.804373 7413 tablet.cc:2378] T 00000000000000000000000000000000 P 98276328da3f46a28846576a2d5ab83e: Can't schedule compaction. Clean time has not been advanced past its initial value.
02:06:03.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:03.804467 7613 sys_catalog.cc:564] T 00000000000000000000000000000000 P 1860f38db9cc4b6bb85fa7f2d490f6b2 [sys.catalog]: configured and running, proceeding with master startup.
02:06:03.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:03.836469 7626 raft_consensus.cc:491] T 00000000000000000000000000000000 P 1860f38db9cc4b6bb85fa7f2d490f6b2 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:06:03.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:03.837039 7626 raft_consensus.cc:513] T 00000000000000000000000000000000 P 1860f38db9cc4b6bb85fa7f2d490f6b2 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "98276328da3f46a28846576a2d5ab83e" member_type: VOTER last_known_addr { host: "127.7.52.254" port: 46311 } } peers { permanent_uuid: "dbbf338bb3b94a36aa32a3103260e759" member_type: VOTER last_known_addr { host: "127.7.52.253" port: 41921 } } peers { permanent_uuid: "1860f38db9cc4b6bb85fa7f2d490f6b2" member_type: VOTER last_known_addr { host: "127.7.52.252" port: 43529 } }
02:06:03.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:03.840101 7626 leader_election.cc:290] T 00000000000000000000000000000000 P 1860f38db9cc4b6bb85fa7f2d490f6b2 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 98276328da3f46a28846576a2d5ab83e (127.7.52.254:46311), dbbf338bb3b94a36aa32a3103260e759 (127.7.52.253:41921)
02:06:03.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:03.846939 7434 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "1860f38db9cc4b6bb85fa7f2d490f6b2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "98276328da3f46a28846576a2d5ab83e" is_pre_election: true
02:06:03.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:03.848176 7434 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 98276328da3f46a28846576a2d5ab83e [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 1860f38db9cc4b6bb85fa7f2d490f6b2 in term 0.
02:06:03.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:03.858233 7567 tablet.cc:2378] T 00000000000000000000000000000000 P 1860f38db9cc4b6bb85fa7f2d490f6b2: Can't schedule compaction. Clean time has not been advanced past its initial value.
02:06:03.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:03.860528 7648 catalog_manager.cc:1560] T 00000000000000000000000000000000 P 1860f38db9cc4b6bb85fa7f2d490f6b2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:06:03.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:03.861171 7648 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:06:03.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:03.862525 7562 leader_election.cc:304] T 00000000000000000000000000000000 P 1860f38db9cc4b6bb85fa7f2d490f6b2 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1860f38db9cc4b6bb85fa7f2d490f6b2, 98276328da3f46a28846576a2d5ab83e; no voters:
02:06:03.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:03.863572 7626 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 1860f38db9cc4b6bb85fa7f2d490f6b2 [term 0 FOLLOWER]: Leader pre-election won for term 1
02:06:03.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:03.863922 7626 raft_consensus.cc:491] T 00000000000000000000000000000000 P 1860f38db9cc4b6bb85fa7f2d490f6b2 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
02:06:03.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:03.864262 7626 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 1860f38db9cc4b6bb85fa7f2d490f6b2 [term 0 FOLLOWER]: Advancing to term 1
02:06:03.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:03.874349 7626 raft_consensus.cc:513] T 00000000000000000000000000000000 P 1860f38db9cc4b6bb85fa7f2d490f6b2 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "98276328da3f46a28846576a2d5ab83e" member_type: VOTER last_known_addr { host: "127.7.52.254" port: 46311 } } peers { permanent_uuid: "dbbf338bb3b94a36aa32a3103260e759" member_type: VOTER last_known_addr { host: "127.7.52.253" port: 41921 } } peers { permanent_uuid: "1860f38db9cc4b6bb85fa7f2d490f6b2" member_type: VOTER last_known_addr { host: "127.7.52.252" port: 43529 } }
02:06:03.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:03.879226 7434 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "1860f38db9cc4b6bb85fa7f2d490f6b2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "98276328da3f46a28846576a2d5ab83e"
02:06:03.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:03.880213 7434 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 98276328da3f46a28846576a2d5ab83e [term 0 FOLLOWER]: Advancing to term 1
02:06:03.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:03.892354 7626 leader_election.cc:290] T 00000000000000000000000000000000 P 1860f38db9cc4b6bb85fa7f2d490f6b2 [CANDIDATE]: Term 1 election: Requested vote from peers 98276328da3f46a28846576a2d5ab83e (127.7.52.254:46311), dbbf338bb3b94a36aa32a3103260e759 (127.7.52.253:41921)
02:06:03.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:03.901268 7434 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 98276328da3f46a28846576a2d5ab83e [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 1860f38db9cc4b6bb85fa7f2d490f6b2 in term 1.
02:06:03.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:03.903473 7562 leader_election.cc:304] T 00000000000000000000000000000000 P 1860f38db9cc4b6bb85fa7f2d490f6b2 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1860f38db9cc4b6bb85fa7f2d490f6b2, 98276328da3f46a28846576a2d5ab83e; no voters:
02:06:03.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:03.904811 7626 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 1860f38db9cc4b6bb85fa7f2d490f6b2 [term 1 FOLLOWER]: Leader election won for term 1
02:06:03.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:03.915241 7626 raft_consensus.cc:695] T 00000000000000000000000000000000 P 1860f38db9cc4b6bb85fa7f2d490f6b2 [term 1 LEADER]: Becoming Leader. State: Replica: 1860f38db9cc4b6bb85fa7f2d490f6b2, State: Running, Role: LEADER
02:06:03.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:03.917263 7626 consensus_queue.cc:237] T 00000000000000000000000000000000 P 1860f38db9cc4b6bb85fa7f2d490f6b2 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "98276328da3f46a28846576a2d5ab83e" member_type: VOTER last_known_addr { host: "127.7.52.254" port: 46311 } } peers { permanent_uuid: "dbbf338bb3b94a36aa32a3103260e759" member_type: VOTER last_known_addr { host: "127.7.52.253" port: 41921 } } peers { permanent_uuid: "1860f38db9cc4b6bb85fa7f2d490f6b2" member_type: VOTER last_known_addr { host: "127.7.52.252" port: 43529 } }
02:06:03.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:03.932091 7647 catalog_manager.cc:1560] T 00000000000000000000000000000000 P 98276328da3f46a28846576a2d5ab83e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:06:03.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:03.933846 7647 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:06:03.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:03.946012 7651 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1860f38db9cc4b6bb85fa7f2d490f6b2 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 1860f38db9cc4b6bb85fa7f2d490f6b2. Latest consensus state: current_term: 1 leader_uuid: "1860f38db9cc4b6bb85fa7f2d490f6b2" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "98276328da3f46a28846576a2d5ab83e" member_type: VOTER last_known_addr { host: "127.7.52.254" port: 46311 } } peers { permanent_uuid: "dbbf338bb3b94a36aa32a3103260e759" member_type: VOTER last_known_addr { host: "127.7.52.253" port: 41921 } } peers { permanent_uuid: "1860f38db9cc4b6bb85fa7f2d490f6b2" member_type: VOTER last_known_addr { host: "127.7.52.252" port: 43529 } } }
02:06:03.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:03.946875 7651 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1860f38db9cc4b6bb85fa7f2d490f6b2 [sys.catalog]: This master's current role is: LEADER
02:06:03.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:03.957639 7653 catalog_manager.cc:1477] Loading table and tablet metadata into memory...
02:06:03.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:03.839480 7391 master_runner.cc:186] Error getting master registration for 127.7.52.253:41921: OK, instance_id { permanent_uuid: "dbbf338bb3b94a36aa32a3103260e759" instance_seqno: 1751076361877901 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
02:06:03.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:03.974398 7391 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
02:06:03.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:03.979938 7653 catalog_manager.cc:1486] Initializing Kudu cluster ID...
02:06:04.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:03.855309 7536 master_runner.cc:186] Error getting master registration for 127.7.52.253:41921: OK, instance_id { permanent_uuid: "dbbf338bb3b94a36aa32a3103260e759" instance_seqno: 1751076361877901 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
02:06:04.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:04.000666 7536 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
02:06:04.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:04.018705 7434 raft_consensus.cc:1273] T 00000000000000000000000000000000 P 98276328da3f46a28846576a2d5ab83e [term 1 FOLLOWER]: Refusing update from remote peer 1860f38db9cc4b6bb85fa7f2d490f6b2: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
02:06:04.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:04.022650 7651 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 1860f38db9cc4b6bb85fa7f2d490f6b2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "98276328da3f46a28846576a2d5ab83e" member_type: VOTER last_known_addr { host: "127.7.52.254" port: 46311 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
02:06:04.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:04.055435 7625 sys_catalog.cc:455] T 00000000000000000000000000000000 P 98276328da3f46a28846576a2d5ab83e [sys.catalog]: SysCatalogTable state changed. Reason: New leader 1860f38db9cc4b6bb85fa7f2d490f6b2. Latest consensus state: current_term: 1 leader_uuid: "1860f38db9cc4b6bb85fa7f2d490f6b2" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "98276328da3f46a28846576a2d5ab83e" member_type: VOTER last_known_addr { host: "127.7.52.254" port: 46311 } } peers { permanent_uuid: "dbbf338bb3b94a36aa32a3103260e759" member_type: VOTER last_known_addr { host: "127.7.52.253" port: 41921 } } peers { permanent_uuid: "1860f38db9cc4b6bb85fa7f2d490f6b2" member_type: VOTER last_known_addr { host: "127.7.52.252" port: 43529 } } }
02:06:04.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:04.056317 7625 sys_catalog.cc:458] T 00000000000000000000000000000000 P 98276328da3f46a28846576a2d5ab83e [sys.catalog]: This master's current role is: FOLLOWER
02:06:04.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:04.061264 7654 mvcc.cc:204] Tried to move back new op lower bound from 7172408786985922560 to 7172408786636713984. Current Snapshot: MvccSnapshot[applied={T|T < 7172408786985922560}]
02:06:04.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:04.062589 7651 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1860f38db9cc4b6bb85fa7f2d490f6b2 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "1860f38db9cc4b6bb85fa7f2d490f6b2" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "98276328da3f46a28846576a2d5ab83e" member_type: VOTER last_known_addr { host: "127.7.52.254" port: 46311 } } peers { permanent_uuid: "dbbf338bb3b94a36aa32a3103260e759" member_type: VOTER last_known_addr { host: "127.7.52.253" port: 41921 } } peers { permanent_uuid: "1860f38db9cc4b6bb85fa7f2d490f6b2" member_type: VOTER last_known_addr { host: "127.7.52.252" port: 43529 } } }
02:06:04.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:04.063310 7651 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1860f38db9cc4b6bb85fa7f2d490f6b2 [sys.catalog]: This master's current role is: LEADER
02:06:04.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:04.069960 7653 catalog_manager.cc:1349] Generated new cluster ID: 8b94064aa4dc46d08116ff53aee97bc3
02:06:04.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:04.070264 7653 catalog_manager.cc:1497] Initializing Kudu internal certificate authority...
02:06:04.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:04.086174 7625 sys_catalog.cc:455] T 00000000000000000000000000000000 P 98276328da3f46a28846576a2d5ab83e [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "1860f38db9cc4b6bb85fa7f2d490f6b2" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "98276328da3f46a28846576a2d5ab83e" member_type: VOTER last_known_addr { host: "127.7.52.254" port: 46311 } } peers { permanent_uuid: "dbbf338bb3b94a36aa32a3103260e759" member_type: VOTER last_known_addr { host: "127.7.52.253" port: 41921 } } peers { permanent_uuid: "1860f38db9cc4b6bb85fa7f2d490f6b2" member_type: VOTER last_known_addr { host: "127.7.52.252" port: 43529 } } }
02:06:04.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:04.086946 7625 sys_catalog.cc:458] T 00000000000000000000000000000000 P 98276328da3f46a28846576a2d5ab83e [sys.catalog]: This master's current role is: FOLLOWER
02:06:04.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:04.130005 7653 catalog_manager.cc:1372] Generated new certificate authority record
02:06:04.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:04.138247 7653 catalog_manager.cc:1506] Loading token signing keys...
02:06:04.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:04.211815 7653 catalog_manager.cc:5955] T 00000000000000000000000000000000 P 1860f38db9cc4b6bb85fa7f2d490f6b2: Generated new TSK 0
02:06:04.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:04.214022 7653 catalog_manager.cc:1516] Initializing in-progress tserver states...
02:06:04.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:04.238032 7379 external_mini_cluster.cc:1351] Running /tmp/dist-test-task8JcgIg/build/tsan/bin/kudu
02:06:04.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task8JcgIg/build/tsan/bin/kudu
02:06:04.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster5757810492041090520/ts-0/wal
02:06:04.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster5757810492041090520/ts-0/data
02:06:04.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:06:04.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:06:04.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:06:04.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:06:04.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:06:04.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:06:04.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster5757810492041090520/ts-0/logs
02:06:04.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster5757810492041090520/ts-0/data/info.pb
02:06:04.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:06:04.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:06:04.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:06:04.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:06:04.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:06:04.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:06:04.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:06:04.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:06:04.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
02:06:04.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:06:04.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.52.193:0
02:06:04.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.7.52.193
02:06:04.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.52.193
02:06:04.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:06:04.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.7.52.254:46311,127.7.52.253:41921,127.7.52.252:43529
02:06:04.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.52.212:44749
02:06:04.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:06:04.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:06:04.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
02:06:04.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:04.248431 7534 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.52.252" port: 43529 } attempt: 7
02:06:04.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:04.304512 7534 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P dbbf338bb3b94a36aa32a3103260e759: Bootstrap starting.
02:06:04.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:04.325281 7534 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P dbbf338bb3b94a36aa32a3103260e759: Neither blocks nor log segments found. Creating new log.
02:06:04.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:04.331434 7534 log.cc:826] T 00000000000000000000000000000000 P dbbf338bb3b94a36aa32a3103260e759: Log is configured to *not* fsync() on all Append() calls
02:06:04.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:04.344586 7534 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P dbbf338bb3b94a36aa32a3103260e759: No bootstrap required, opened a new log
02:06:04.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:04.424603 7534 raft_consensus.cc:357] T 00000000000000000000000000000000 P dbbf338bb3b94a36aa32a3103260e759 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "98276328da3f46a28846576a2d5ab83e" member_type: VOTER last_known_addr { host: "127.7.52.254" port: 46311 } } peers { permanent_uuid: "dbbf338bb3b94a36aa32a3103260e759" member_type: VOTER last_known_addr { host: "127.7.52.253" port: 41921 } } peers { permanent_uuid: "1860f38db9cc4b6bb85fa7f2d490f6b2" member_type: VOTER last_known_addr { host: "127.7.52.252" port: 43529 } }
02:06:04.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:04.427309 7534 raft_consensus.cc:383] T 00000000000000000000000000000000 P dbbf338bb3b94a36aa32a3103260e759 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:06:04.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:04.428391 7534 raft_consensus.cc:738] T 00000000000000000000000000000000 P dbbf338bb3b94a36aa32a3103260e759 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: dbbf338bb3b94a36aa32a3103260e759, State: Initialized, Role: FOLLOWER
02:06:04.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:04.431102 7534 consensus_queue.cc:260] T 00000000000000000000000000000000 P dbbf338bb3b94a36aa32a3103260e759 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "98276328da3f46a28846576a2d5ab83e" member_type: VOTER last_known_addr { host: "127.7.52.254" port: 46311 } } peers { permanent_uuid: "dbbf338bb3b94a36aa32a3103260e759" member_type: VOTER last_known_addr { host: "127.7.52.253" port: 41921 } } peers { permanent_uuid: "1860f38db9cc4b6bb85fa7f2d490f6b2" member_type: VOTER last_known_addr { host: "127.7.52.252" port: 43529 } }
02:06:04.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:04.453190 7663 sys_catalog.cc:455] T 00000000000000000000000000000000 P dbbf338bb3b94a36aa32a3103260e759 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "98276328da3f46a28846576a2d5ab83e" member_type: VOTER last_known_addr { host: "127.7.52.254" port: 46311 } } peers { permanent_uuid: "dbbf338bb3b94a36aa32a3103260e759" member_type: VOTER last_known_addr { host: "127.7.52.253" port: 41921 } } peers { permanent_uuid: "1860f38db9cc4b6bb85fa7f2d490f6b2" member_type: VOTER last_known_addr { host: "127.7.52.252" port: 43529 } } }
02:06:04.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:04.454061 7663 sys_catalog.cc:458] T 00000000000000000000000000000000 P dbbf338bb3b94a36aa32a3103260e759 [sys.catalog]: This master's current role is: FOLLOWER
02:06:04.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:04.463964 7507 raft_consensus.cc:3058] T 00000000000000000000000000000000 P dbbf338bb3b94a36aa32a3103260e759 [term 0 FOLLOWER]: Advancing to term 1
02:06:04.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:04.467211 7508 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "1860f38db9cc4b6bb85fa7f2d490f6b2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "dbbf338bb3b94a36aa32a3103260e759"
02:06:04.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:04.468804 7509 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "1860f38db9cc4b6bb85fa7f2d490f6b2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "dbbf338bb3b94a36aa32a3103260e759" is_pre_election: true
02:06:04.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:04.474678 7507 raft_consensus.cc:1273] T 00000000000000000000000000000000 P dbbf338bb3b94a36aa32a3103260e759 [term 1 FOLLOWER]: Refusing update from remote peer 1860f38db9cc4b6bb85fa7f2d490f6b2: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
02:06:04.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:04.476416 7534 sys_catalog.cc:564] T 00000000000000000000000000000000 P dbbf338bb3b94a36aa32a3103260e759 [sys.catalog]: configured and running, proceeding with master startup.
02:06:04.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:04.477105 7651 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 1860f38db9cc4b6bb85fa7f2d490f6b2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "dbbf338bb3b94a36aa32a3103260e759" member_type: VOTER last_known_addr { host: "127.7.52.253" port: 41921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
02:06:04.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:04.561357 7663 sys_catalog.cc:455] T 00000000000000000000000000000000 P dbbf338bb3b94a36aa32a3103260e759 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 1860f38db9cc4b6bb85fa7f2d490f6b2. Latest consensus state: current_term: 1 leader_uuid: "1860f38db9cc4b6bb85fa7f2d490f6b2" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "98276328da3f46a28846576a2d5ab83e" member_type: VOTER last_known_addr { host: "127.7.52.254" port: 46311 } } peers { permanent_uuid: "dbbf338bb3b94a36aa32a3103260e759" member_type: VOTER last_known_addr { host: "127.7.52.253" port: 41921 } } peers { permanent_uuid: "1860f38db9cc4b6bb85fa7f2d490f6b2" member_type: VOTER last_known_addr { host: "127.7.52.252" port: 43529 } } }
02:06:04.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:04.562156 7663 sys_catalog.cc:458] T 00000000000000000000000000000000 P dbbf338bb3b94a36aa32a3103260e759 [sys.catalog]: This master's current role is: FOLLOWER
02:06:04.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:04.561657 7669 sys_catalog.cc:455] T 00000000000000000000000000000000 P dbbf338bb3b94a36aa32a3103260e759 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "1860f38db9cc4b6bb85fa7f2d490f6b2" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "98276328da3f46a28846576a2d5ab83e" member_type: VOTER last_known_addr { host: "127.7.52.254" port: 46311 } } peers { permanent_uuid: "dbbf338bb3b94a36aa32a3103260e759" member_type: VOTER last_known_addr { host: "127.7.52.253" port: 41921 } } peers { permanent_uuid: "1860f38db9cc4b6bb85fa7f2d490f6b2" member_type: VOTER last_known_addr { host: "127.7.52.252" port: 43529 } } }
02:06:04.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:04.563256 7669 sys_catalog.cc:458] T 00000000000000000000000000000000 P dbbf338bb3b94a36aa32a3103260e759 [sys.catalog]: This master's current role is: FOLLOWER
02:06:04.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:04.640975 7679 catalog_manager.cc:1261] Loaded cluster ID: 8b94064aa4dc46d08116ff53aee97bc3
02:06:04.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:04.641723 7679 catalog_manager.cc:1554] T 00000000000000000000000000000000 P dbbf338bb3b94a36aa32a3103260e759: loading cluster ID for follower catalog manager: success
02:06:04.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:04.649607 7679 catalog_manager.cc:1576] T 00000000000000000000000000000000 P dbbf338bb3b94a36aa32a3103260e759: acquiring CA information for follower catalog manager: success
02:06:04.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:04.655776 7679 catalog_manager.cc:1604] T 00000000000000000000000000000000 P dbbf338bb3b94a36aa32a3103260e759: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
02:06:04.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:04.874488 7659 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:06:04.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:04.875145 7659 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:06:04.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:04.875934 7659 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:06:04.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:04.944944 7647 catalog_manager.cc:1261] Loaded cluster ID: 8b94064aa4dc46d08116ff53aee97bc3
02:06:04.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:04.945654 7647 catalog_manager.cc:1554] T 00000000000000000000000000000000 P 98276328da3f46a28846576a2d5ab83e: loading cluster ID for follower catalog manager: success
02:06:04.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:04.965220 7647 catalog_manager.cc:1576] T 00000000000000000000000000000000 P 98276328da3f46a28846576a2d5ab83e: acquiring CA information for follower catalog manager: success
02:06:04.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:04.971513 7647 catalog_manager.cc:1604] T 00000000000000000000000000000000 P 98276328da3f46a28846576a2d5ab83e: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
02:06:04.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:04.977470 7659 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:06:04.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:04.979017 7659 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.7.52.193
02:06:05.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:05.019464 7651 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1860f38db9cc4b6bb85fa7f2d490f6b2 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "1860f38db9cc4b6bb85fa7f2d490f6b2" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "98276328da3f46a28846576a2d5ab83e" member_type: VOTER last_known_addr { host: "127.7.52.254" port: 46311 } } peers { permanent_uuid: "dbbf338bb3b94a36aa32a3103260e759" member_type: VOTER last_known_addr { host: "127.7.52.253" port: 41921 } } peers { permanent_uuid: "1860f38db9cc4b6bb85fa7f2d490f6b2" member_type: VOTER last_known_addr { host: "127.7.52.252" port: 43529 } } }
02:06:05.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:05.021665 7651 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1860f38db9cc4b6bb85fa7f2d490f6b2 [sys.catalog]: This master's current role is: LEADER
02:06:05.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:05.059753 7659 tablet_server_runner.cc:78] Tablet server non-default flags:
02:06:05.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:06:05.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.52.212:44749
02:06:05.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:06:05.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:06:05.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster5757810492041090520/ts-0/data
02:06:05.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster5757810492041090520/ts-0/wal
02:06:05.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:06:05.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:06:05.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.52.193:0
02:06:05.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:06:05.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:06:05.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:06:05.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster5757810492041090520/ts-0/data/info.pb
02:06:05.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.52.193
02:06:05.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:06:05.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.7.52.254:46311,127.7.52.253:41921,127.7.52.252:43529
02:06:05.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:06:05.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:06:05.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:06:05.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:06:05.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:06:05.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.7.52.193
02:06:05.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster5757810492041090520/ts-0/logs
02:06:05.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:06:05.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:06:05.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:06:05.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
02:06:05.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
02:06:05.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision f0776864a65c19ebdd909053f3e624e8bcc25cee
02:06:05.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:06:05.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 28 Jun 2025 01:43:19 UTC on 5fd53c4cbb9d
02:06:05.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 6805
02:06:05.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:06:05.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:05.063580 7659 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:06:05.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:05.066167 7659 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:06:05.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:05.114910 7690 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:07.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:06.498991 7688 debug-util.cc:398] Leaking SignalData structure 0x7b0800000a80 after lost signal to thread 7659
02:06:07.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:07.248261 7659 thread.cc:640] GCE (cloud detector) Time spent creating pthread: real 2.141s user 0.425s sys 0.835s
02:06:07.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:07.248679 7659 thread.cc:606] GCE (cloud detector) Time spent starting thread: real 2.141s user 0.425s sys 0.835s
02:06:07.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:07.259068 7693 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:07.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:05.117949 7689 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:07.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:07.266933 7659 server_base.cc:1048] running on GCE node
02:06:07.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:07.268326 7659 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:06:07.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:07.271023 7659 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:06:07.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:07.272575 7659 hybrid_clock.cc:648] HybridClock initialized: now 1751076367272433 us; error 157 us; skew 500 ppm
02:06:07.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:07.273438 7659 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:06:07.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:07.285794 7659 webserver.cc:469] Webserver started at http://127.7.52.193:39717/ using document root <none> and password file <none>
02:06:07.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:07.287006 7659 fs_manager.cc:362] Metadata directory not provided
02:06:07.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:07.287251 7659 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:06:07.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:07.287768 7659 server_base.cc:896] This appears to be a new deployment of Kudu; creating new FS layout
02:06:07.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:07.292415 7659 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster5757810492041090520/ts-0/data/instance:
02:06:07.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cdb0898087cb4db4816ae7e8552f6eaf"
02:06:07.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-28 02:06:07 on dist-test-slave-lq1z"
02:06:07.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:07.294083 7659 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster5757810492041090520/ts-0/wal/instance:
02:06:07.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cdb0898087cb4db4816ae7e8552f6eaf"
02:06:07.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-28 02:06:07 on dist-test-slave-lq1z"
02:06:07.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:07.302930 7659 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.003s sys 0.007s
02:06:07.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:07.310734 7699 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:06:07.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:07.312031 7659 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.001s sys 0.003s
02:06:07.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:07.312446 7659 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster5757810492041090520/ts-0/data,/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster5757810492041090520/ts-0/wal
02:06:07.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cdb0898087cb4db4816ae7e8552f6eaf"
02:06:07.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-28 02:06:07 on dist-test-slave-lq1z"
02:06:07.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:07.312880 7659 fs_report.cc:389] FS layout report
02:06:07.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:06:07.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster5757810492041090520/ts-0/wal
02:06:07.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster5757810492041090520/ts-0/wal
02:06:07.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster5757810492041090520/ts-0/data/data
02:06:07.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:06:07.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:06:07.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:06:07.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:06:07.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:06:07.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:06:07.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:06:07.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:06:07.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:06:07.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:06:07.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:06:07.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:07.385124 7659 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:06:07.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:07.387274 7659 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:06:07.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:07.387907 7659 kserver.cc:163] Server-wide thread pool size limit: 3276
02:06:07.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:07.392108 7659 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
02:06:07.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:07.398603 7659 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
02:06:07.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:07.398912 7659 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
02:06:07.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:07.399197 7659 ts_tablet_manager.cc:610] Registered 0 tablets
02:06:07.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:07.399430 7659 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
02:06:07.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:07.622829 7659 rpc_server.cc:307] RPC server started. Bound to: 127.7.52.193:46019
02:06:07.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:07.623873 7811 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.7.52.193:46019 every 8 connection(s)
02:06:07.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:07.626457 7659 server_base.cc:1180] Dumped server information to /tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster5757810492041090520/ts-0/data/info.pb
02:06:07.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:07.633912 7379 external_mini_cluster.cc:1413] Started /tmp/dist-test-task8JcgIg/build/tsan/bin/kudu as pid 7659
02:06:07.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:07.634548 7379 external_mini_cluster.cc:1427] Reading /tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster5757810492041090520/ts-0/wal/instance
02:06:07.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:07.661913 7379 external_mini_cluster.cc:1351] Running /tmp/dist-test-task8JcgIg/build/tsan/bin/kudu
02:06:07.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task8JcgIg/build/tsan/bin/kudu
02:06:07.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster5757810492041090520/ts-1/wal
02:06:07.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster5757810492041090520/ts-1/data
02:06:07.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:06:07.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:06:07.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:06:07.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:06:07.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:06:07.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:06:07.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster5757810492041090520/ts-1/logs
02:06:07.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster5757810492041090520/ts-1/data/info.pb
02:06:07.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:06:07.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:06:07.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:06:07.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:06:07.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:06:07.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:06:07.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:06:07.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:06:07.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
02:06:07.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:06:07.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.52.194:0
02:06:07.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.7.52.194
02:06:07.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.52.194
02:06:07.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:06:07.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.7.52.254:46311,127.7.52.253:41921,127.7.52.252:43529
02:06:07.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.52.212:44749
02:06:07.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:06:07.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:06:07.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
02:06:07.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:07.693748 7812 heartbeater.cc:344] Connected to a master server at 127.7.52.252:43529
02:06:07.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:07.694258 7812 heartbeater.cc:461] Registering TS with master...
02:06:07.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:07.695523 7812 heartbeater.cc:507] Master 127.7.52.252:43529 requested a full tablet report, sending...
02:06:07.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:07.698688 7578 ts_manager.cc:194] Registered new tserver with Master: cdb0898087cb4db4816ae7e8552f6eaf (127.7.52.193:46019)
02:06:07.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:07.701721 7578 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.7.52.193:48069
02:06:07.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:07.707046 7813 heartbeater.cc:344] Connected to a master server at 127.7.52.253:41921
02:06:07.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:07.707549 7813 heartbeater.cc:461] Registering TS with master...
02:06:07.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:07.708447 7813 heartbeater.cc:507] Master 127.7.52.253:41921 requested a full tablet report, sending...
02:06:07.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:07.712551 7499 ts_manager.cc:194] Registered new tserver with Master: cdb0898087cb4db4816ae7e8552f6eaf (127.7.52.193:46019)
02:06:07.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:07.744807 7816 heartbeater.cc:344] Connected to a master server at 127.7.52.254:46311
02:06:07.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:07.745193 7816 heartbeater.cc:461] Registering TS with master...
02:06:07.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:07.745847 7816 heartbeater.cc:507] Master 127.7.52.254:46311 requested a full tablet report, sending...
02:06:07.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:07.748602 7423 ts_manager.cc:194] Registered new tserver with Master: cdb0898087cb4db4816ae7e8552f6eaf (127.7.52.193:46019)
02:06:08.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:08.056937 7819 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:06:08.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:08.057498 7819 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:06:08.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:08.058192 7819 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:06:08.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:08.091511 7819 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:06:08.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:08.092540 7819 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.7.52.194
02:06:08.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:08.128580 7819 tablet_server_runner.cc:78] Tablet server non-default flags:
02:06:08.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:06:08.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.52.212:44749
02:06:08.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:06:08.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:06:08.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster5757810492041090520/ts-1/data
02:06:08.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster5757810492041090520/ts-1/wal
02:06:08.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:06:08.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:06:08.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.52.194:0
02:06:08.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:06:08.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:06:08.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:06:08.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster5757810492041090520/ts-1/data/info.pb
02:06:08.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.52.194
02:06:08.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:06:08.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.7.52.254:46311,127.7.52.253:41921,127.7.52.252:43529
02:06:08.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:06:08.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:06:08.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:06:08.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:06:08.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:06:08.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.7.52.194
02:06:08.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster5757810492041090520/ts-1/logs
02:06:08.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:06:08.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:06:08.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:06:08.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
02:06:08.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
02:06:08.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision f0776864a65c19ebdd909053f3e624e8bcc25cee
02:06:08.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:06:08.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 28 Jun 2025 01:43:19 UTC on 5fd53c4cbb9d
02:06:08.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 6805
02:06:08.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:06:08.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:08.130312 7819 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:06:08.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:08.132458 7819 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:06:08.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:08.152539 7827 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:08.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:08.706801 7812 heartbeater.cc:499] Master 127.7.52.252:43529 was elected leader, sending a full tablet report...
02:06:09.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:08.152791 7828 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:09.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:08.153460 7830 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:09.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:09.337881 7829 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
02:06:09.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:09.337934 7819 server_base.cc:1043] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:06:09.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:09.342162 7819 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:06:09.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:09.344897 7819 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:06:09.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:09.346318 7819 hybrid_clock.cc:648] HybridClock initialized: now 1751076369346279 us; error 58 us; skew 500 ppm
02:06:09.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:09.347077 7819 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:06:09.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:09.353724 7819 webserver.cc:469] Webserver started at http://127.7.52.194:40299/ using document root <none> and password file <none>
02:06:09.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:09.354589 7819 fs_manager.cc:362] Metadata directory not provided
02:06:09.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:09.354776 7819 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:06:09.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:09.355154 7819 server_base.cc:896] This appears to be a new deployment of Kudu; creating new FS layout
02:06:09.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:09.359392 7819 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster5757810492041090520/ts-1/data/instance:
02:06:09.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f76ea9265e7746cd9694e94efa53febd"
02:06:09.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-28 02:06:09 on dist-test-slave-lq1z"
02:06:09.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:09.360479 7819 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster5757810492041090520/ts-1/wal/instance:
02:06:09.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f76ea9265e7746cd9694e94efa53febd"
02:06:09.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-28 02:06:09 on dist-test-slave-lq1z"
02:06:09.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:09.367131 7819 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.006s sys 0.000s
02:06:09.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:09.372558 7839 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:06:09.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:09.373556 7819 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.004s sys 0.000s
02:06:09.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:09.373885 7819 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster5757810492041090520/ts-1/data,/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster5757810492041090520/ts-1/wal
02:06:09.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f76ea9265e7746cd9694e94efa53febd"
02:06:09.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-28 02:06:09 on dist-test-slave-lq1z"
02:06:09.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:09.374219 7819 fs_report.cc:389] FS layout report
02:06:09.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:06:09.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster5757810492041090520/ts-1/wal
02:06:09.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster5757810492041090520/ts-1/wal
02:06:09.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster5757810492041090520/ts-1/data/data
02:06:09.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:06:09.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:06:09.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:06:09.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:06:09.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:06:09.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:06:09.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:06:09.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:06:09.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:06:09.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:06:09.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:06:09.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:09.427096 7819 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:06:09.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:09.429136 7819 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:06:09.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:09.429541 7819 kserver.cc:163] Server-wide thread pool size limit: 3276
02:06:09.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:09.431913 7819 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
02:06:09.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:09.435822 7819 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
02:06:09.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:09.436009 7819 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
02:06:09.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:09.436290 7819 ts_tablet_manager.cc:610] Registered 0 tablets
02:06:09.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:09.436455 7819 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
02:06:09.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:09.597143 7819 rpc_server.cc:307] RPC server started. Bound to: 127.7.52.194:37987
02:06:09.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:09.597245 7951 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.7.52.194:37987 every 8 connection(s)
02:06:09.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:09.599779 7819 server_base.cc:1180] Dumped server information to /tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster5757810492041090520/ts-1/data/info.pb
02:06:09.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:09.610625 7379 external_mini_cluster.cc:1413] Started /tmp/dist-test-task8JcgIg/build/tsan/bin/kudu as pid 7819
02:06:09.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:09.611438 7379 external_mini_cluster.cc:1427] Reading /tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster5757810492041090520/ts-1/wal/instance
02:06:09.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:09.633996 7379 external_mini_cluster.cc:1351] Running /tmp/dist-test-task8JcgIg/build/tsan/bin/kudu
02:06:09.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task8JcgIg/build/tsan/bin/kudu
02:06:09.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster5757810492041090520/ts-2/wal
02:06:09.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster5757810492041090520/ts-2/data
02:06:09.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:06:09.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:06:09.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:06:09.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:06:09.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:06:09.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:06:09.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster5757810492041090520/ts-2/logs
02:06:09.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster5757810492041090520/ts-2/data/info.pb
02:06:09.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:06:09.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:06:09.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:06:09.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:06:09.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:06:09.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:06:09.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:06:09.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:06:09.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
02:06:09.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:06:09.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.52.195:0
02:06:09.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.7.52.195
02:06:09.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.52.195
02:06:09.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:06:09.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.7.52.254:46311,127.7.52.253:41921,127.7.52.252:43529
02:06:09.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.52.212:44749
02:06:09.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:06:09.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:06:09.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
02:06:09.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:09.660388 7952 heartbeater.cc:344] Connected to a master server at 127.7.52.252:43529
02:06:09.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:09.660995 7952 heartbeater.cc:461] Registering TS with master...
02:06:09.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:09.662310 7952 heartbeater.cc:507] Master 127.7.52.252:43529 requested a full tablet report, sending...
02:06:09.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:09.666086 7578 ts_manager.cc:194] Registered new tserver with Master: f76ea9265e7746cd9694e94efa53febd (127.7.52.194:37987)
02:06:09.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:09.667744 7578 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.7.52.194:48181
02:06:09.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:09.668933 7954 heartbeater.cc:344] Connected to a master server at 127.7.52.254:46311
02:06:09.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:09.669263 7954 heartbeater.cc:461] Registering TS with master...
02:06:09.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:09.670202 7953 heartbeater.cc:344] Connected to a master server at 127.7.52.253:41921
02:06:09.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:09.670157 7954 heartbeater.cc:507] Master 127.7.52.254:46311 requested a full tablet report, sending...
02:06:09.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:09.670576 7953 heartbeater.cc:461] Registering TS with master...
02:06:09.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:09.671717 7953 heartbeater.cc:507] Master 127.7.52.253:41921 requested a full tablet report, sending...
02:06:09.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:09.673890 7423 ts_manager.cc:194] Registered new tserver with Master: f76ea9265e7746cd9694e94efa53febd (127.7.52.194:37987)
02:06:09.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:09.674162 7499 ts_manager.cc:194] Registered new tserver with Master: f76ea9265e7746cd9694e94efa53febd (127.7.52.194:37987)
02:06:09.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:09.969205 7962 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:06:09.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:09.969692 7962 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:06:09.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:09.970199 7962 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:06:10.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:10.001943 7962 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:06:10.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:10.002743 7962 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.7.52.195
02:06:10.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:10.038125 7962 tablet_server_runner.cc:78] Tablet server non-default flags:
02:06:10.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:06:10.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.52.212:44749
02:06:10.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:06:10.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:06:10.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster5757810492041090520/ts-2/data
02:06:10.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster5757810492041090520/ts-2/wal
02:06:10.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:06:10.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:06:10.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.52.195:0
02:06:10.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:06:10.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:06:10.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:06:10.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster5757810492041090520/ts-2/data/info.pb
02:06:10.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.52.195
02:06:10.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:06:10.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.7.52.254:46311,127.7.52.253:41921,127.7.52.252:43529
02:06:10.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:06:10.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:06:10.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:06:10.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:06:10.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:06:10.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.7.52.195
02:06:10.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster5757810492041090520/ts-2/logs
02:06:10.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:06:10.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:06:10.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:06:10.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
02:06:10.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
02:06:10.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision f0776864a65c19ebdd909053f3e624e8bcc25cee
02:06:10.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:06:10.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 28 Jun 2025 01:43:19 UTC on 5fd53c4cbb9d
02:06:10.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 6805
02:06:10.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:06:10.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:10.039621 7962 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:06:10.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:10.041935 7962 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:06:10.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:10.060880 7968 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:10.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:10.672401 7952 heartbeater.cc:499] Master 127.7.52.252:43529 was elected leader, sending a full tablet report...
02:06:11.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:10.062681 7971 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:11.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:10.060972 7969 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:11.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:11.250317 7970 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
02:06:11.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:11.250361 7962 server_base.cc:1043] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:06:11.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:11.254566 7962 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:06:11.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:11.257386 7962 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:06:11.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:11.258833 7962 hybrid_clock.cc:648] HybridClock initialized: now 1751076371258793 us; error 61 us; skew 500 ppm
02:06:11.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:11.259604 7962 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:06:11.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:11.266319 7962 webserver.cc:469] Webserver started at http://127.7.52.195:40987/ using document root <none> and password file <none>
02:06:11.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:11.267252 7962 fs_manager.cc:362] Metadata directory not provided
02:06:11.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:11.267445 7962 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:06:11.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:11.267825 7962 server_base.cc:896] This appears to be a new deployment of Kudu; creating new FS layout
02:06:11.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:11.272169 7962 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster5757810492041090520/ts-2/data/instance:
02:06:11.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0f2f409f6dd14f0e9400fb9e2b9a896a"
02:06:11.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-28 02:06:11 on dist-test-slave-lq1z"
02:06:11.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:11.273218 7962 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster5757810492041090520/ts-2/wal/instance:
02:06:11.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0f2f409f6dd14f0e9400fb9e2b9a896a"
02:06:11.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-28 02:06:11 on dist-test-slave-lq1z"
02:06:11.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:11.280609 7962 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.002s sys 0.005s
02:06:11.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:11.286228 7980 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:06:11.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:11.287417 7962 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.005s sys 0.001s
02:06:11.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:11.287714 7962 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster5757810492041090520/ts-2/data,/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster5757810492041090520/ts-2/wal
02:06:11.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0f2f409f6dd14f0e9400fb9e2b9a896a"
02:06:11.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-28 02:06:11 on dist-test-slave-lq1z"
02:06:11.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:11.288015 7962 fs_report.cc:389] FS layout report
02:06:11.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:06:11.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster5757810492041090520/ts-2/wal
02:06:11.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster5757810492041090520/ts-2/wal
02:06:11.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster5757810492041090520/ts-2/data/data
02:06:11.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:06:11.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:06:11.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:06:11.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:06:11.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:06:11.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:06:11.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:06:11.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:06:11.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:06:11.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:06:11.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:06:11.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:11.338352 7962 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:06:11.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:11.339782 7962 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:06:11.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:11.340232 7962 kserver.cc:163] Server-wide thread pool size limit: 3276
02:06:11.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:11.342780 7962 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
02:06:11.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:11.346803 7962 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
02:06:11.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:11.346992 7962 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
02:06:11.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:11.347179 7962 ts_tablet_manager.cc:610] Registered 0 tablets
02:06:11.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:11.347313 7962 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
02:06:11.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:11.510905 7962 rpc_server.cc:307] RPC server started. Bound to: 127.7.52.195:39067
02:06:11.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:11.511009 8093 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.7.52.195:39067 every 8 connection(s)
02:06:11.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:11.513614 7962 server_base.cc:1180] Dumped server information to /tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster5757810492041090520/ts-2/data/info.pb
02:06:11.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:11.523067 7379 external_mini_cluster.cc:1413] Started /tmp/dist-test-task8JcgIg/build/tsan/bin/kudu as pid 7962
02:06:11.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:11.523662 7379 external_mini_cluster.cc:1427] Reading /tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster5757810492041090520/ts-2/wal/instance
02:06:11.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:11.568754 8094 heartbeater.cc:344] Connected to a master server at 127.7.52.252:43529
02:06:11.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:11.569260 8094 heartbeater.cc:461] Registering TS with master...
02:06:11.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:11.569792 8096 heartbeater.cc:344] Connected to a master server at 127.7.52.254:46311
02:06:11.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:11.570133 8096 heartbeater.cc:461] Registering TS with master...
02:06:11.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:11.570338 8095 heartbeater.cc:344] Connected to a master server at 127.7.52.253:41921
02:06:11.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:11.570567 8094 heartbeater.cc:507] Master 127.7.52.252:43529 requested a full tablet report, sending...
02:06:11.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:11.570606 8095 heartbeater.cc:461] Registering TS with master...
02:06:11.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:11.571324 8096 heartbeater.cc:507] Master 127.7.52.254:46311 requested a full tablet report, sending...
02:06:11.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:11.571470 8095 heartbeater.cc:507] Master 127.7.52.253:41921 requested a full tablet report, sending...
02:06:11.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:11.572803 7577 ts_manager.cc:194] Registered new tserver with Master: 0f2f409f6dd14f0e9400fb9e2b9a896a (127.7.52.195:39067)
02:06:11.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:11.574182 7423 ts_manager.cc:194] Registered new tserver with Master: 0f2f409f6dd14f0e9400fb9e2b9a896a (127.7.52.195:39067)
02:06:11.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:11.574563 7577 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.7.52.195:53981
02:06:11.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:11.574599 7499 ts_manager.cc:194] Registered new tserver with Master: 0f2f409f6dd14f0e9400fb9e2b9a896a (127.7.52.195:39067)
02:06:11.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:11.595882 7379 external_mini_cluster.cc:934] 3 TS(s) registered with all masters
02:06:11.608 [DEBUG - main] (MiniKuduCluster.java:195) Response:
02:06:11.610 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
02:06:11.623 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.7.52.254"
port: 46311
}
bound_http_address {
host: "127.7.52.254"
port: 33099
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.7.52.253"
port: 41921
}
bound_http_address {
host: "127.7.52.253"
port: 34623
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.7.52.252"
port: 43529
}
bound_http_address {
host: "127.7.52.252"
port: 46085
}
}
}
02:06:11.628 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
02:06:11.632 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.7.52.193"
port: 46019
}
bound_http_address {
host: "127.7.52.193"
port: 39717
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.7.52.194"
port: 37987
}
bound_http_address {
host: "127.7.52.194"
port: 40299
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.7.52.195"
port: 39067
}
bound_http_address {
host: "127.7.52.195"
port: 40987
}
}
}
02:06:11.632 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
02:06:11.801 [DEBUG - main] (CompositeStrategy.java:59) Unable to load modern Subject methods, relying only on legacy methods
java.lang.NoSuchMethodException: javax.security.auth.Subject.current()
at java.lang.Class.getDeclaredMethod(Class.java:2130) ~[?:1.8.0_275]
at org.apache.kudu.client.internals.ModernStrategy.<init>(ModernStrategy.java:51) ~[main/:?]
at org.apache.kudu.client.internals.CompositeStrategy.<init>(CompositeStrategy.java:52) [main/:?]
at org.apache.kudu.client.internals.CompositeStrategy.<clinit>(CompositeStrategy.java:41) [main/:?]
at org.apache.kudu.client.internals.SecurityManagerCompatibility.get(SecurityManagerCompatibility.java:43) [main/:?]
at org.apache.kudu.client.SecurityContext.setupSubject(SecurityContext.java:155) [main/:?]
at org.apache.kudu.client.SecurityContext.<init>(SecurityContext.java:138) [main/:?]
at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:431) [main/:?]
at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:299) [main/:?]
at org.apache.kudu.client.AsyncKuduClient$AsyncKuduClientBuilder.build(AsyncKuduClient.java:3109) [main/:?]
at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:161) [kudu-test-utils-1.18.0-SNAPSHOT-unshaded.jar:1.18.0-SNAPSHOT]
at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50) [junit-4.13.2.jar:4.13.2]
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184) [kudu-test-utils-1.18.0-SNAPSHOT-unshaded.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244) [kudu-test-utils-1.18.0-SNAPSHOT-unshaded.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184) [kudu-test-utils-1.18.0-SNAPSHOT-unshaded.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244) [kudu-test-utils-1.18.0-SNAPSHOT-unshaded.jar:1.18.0-SNAPSHOT]
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.run(ParentRunner.java:413) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.Suite.runChild(Suite.java:128) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.Suite.runChild(Suite.java:27) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.run(ParentRunner.java:413) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.run(JUnitCore.java:137) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.run(JUnitCore.java:115) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.main(JUnitCore.java:36) [junit-4.13.2.jar:4.13.2]
java.lang.NoSuchMethodException: javax.security.auth.Subject.current()
at java.lang.Class.getDeclaredMethod(Class.java:2130)
at org.apache.kudu.client.internals.ModernStrategy.<init>(ModernStrategy.java:51)
at org.apache.kudu.client.internals.CompositeStrategy.<init>(CompositeStrategy.java:52)
at org.apache.kudu.client.internals.CompositeStrategy.<clinit>(CompositeStrategy.java:41)
at org.apache.kudu.client.internals.SecurityManagerCompatibility.get(SecurityManagerCompatibility.java:43)
at org.apache.kudu.client.SecurityContext.setupSubject(SecurityContext.java:155)
at org.apache.kudu.client.SecurityContext.<init>(SecurityContext.java:138)
at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:431)
at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$AsyncKuduClientBuilder.build(AsyncKuduClient.java:3109)
at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:161)
at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runners.Suite.runChild(Suite.java:128)
at org.junit.runners.Suite.runChild(Suite.java:27)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runner.JUnitCore.run(JUnitCore.java:137)
at org.junit.runner.JUnitCore.run(JUnitCore.java:115)
at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77)
at org.junit.runner.JUnitCore.main(JUnitCore.java:36)
02:06:11.826 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
02:06:11.947 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-task8JcgIg/test-tmp
02:06:11.948 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
02:06:11.948 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-task8JcgIg/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
02:06:11.950 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: true
cluster_root: "/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809"
extra_master_flags: "--authn_token_validity_seconds=10"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
02:06:12.292 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:06:12.292 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
02:06:12.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:12.295430 8105 mini_oidc.cc:150] Starting JWKS server
02:06:12.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:12.299077 8105 webserver.cc:469] Webserver started at http://127.0.0.1:35625/ using document root <none> and password file <none>
02:06:12.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:12.299597 8105 mini_oidc.cc:186] Starting OIDC Discovery server
02:06:12.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:12.301374 8105 webserver.cc:469] Webserver started at http://127.0.0.1:41933/ using document root <none> and password file <none>
02:06:12.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
02:06:12.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/krb5kdc/principal' for realm 'KRBTEST.COM',
02:06:12.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
02:06:12.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 28 02:06:12 dist-test-slave-lq1z krb5kdc[8119](info): setting up network...
02:06:12.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
02:06:12.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 28 02:06:12 dist-test-slave-lq1z krb5kdc[8119](info): set up 2 sockets
02:06:12.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 28 02:06:12 dist-test-slave-lq1z krb5kdc[8119](info): commencing operation
02:06:12.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
02:06:12.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:12.577706 8094 heartbeater.cc:499] Master 127.7.52.252:43529 was elected leader, sending a full tablet report...
02:06:14.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:14.423321 8105 mini_kdc.cc:121] Time spent starting KDC: real 2.083s user 0.004s sys 0.016s
02:06:14.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
02:06:14.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
02:06:14.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
02:06:14.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
02:06:14.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
02:06:14.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
02:06:14.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
02:06:14.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
02:06:14.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
02:06:14.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
02:06:14.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/krb5kdc/test-user.keytab.
02:06:14.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/krb5kdc/test-user.keytab.
02:06:14.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 28 02:06:14 dist-test-slave-lq1z krb5kdc[8119](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1751076374, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:06:14.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
02:06:14.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-06-28T02:06:14Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
02:06:14.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-06-28T02:06:14Z Disabled control of system clock
02:06:14.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.7.234.126@KRBTEST.COM; defaulting to no policy
02:06:14.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:06:14.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.7.234.126@KRBTEST.COM" created.
02:06:14.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:06:14.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.7.234.126 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/krb5kdc/kudu_127.7.234.126.keytab.
02:06:14.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.7.234.126 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/krb5kdc/kudu_127.7.234.126.keytab.
02:06:14.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:14.629532 8105 external_mini_cluster.cc:1351] Running /tmp/dist-test-task8JcgIg/build/tsan/bin/kudu
02:06:14.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task8JcgIg/build/tsan/bin/kudu
02:06:14.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/master-0/wal
02:06:14.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/master-0/data
02:06:14.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:06:14.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:06:14.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:06:14.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:06:14.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:06:14.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:06:14.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/master-0/logs
02:06:14.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/master-0/data/info.pb
02:06:14.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:06:14.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:06:14.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:06:14.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:06:14.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:06:14.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:06:14.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:06:14.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:06:14.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:06:14.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:06:14.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:06:14.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:06:14.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.234.126:34333
02:06:14.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.234.126
02:06:14.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:06:14.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/krb5kdc/kudu_127.7.234.126.keytab
02:06:14.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.7.234.126
02:06:14.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:06:14.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
02:06:14.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
02:06:14.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.234.84:33745
02:06:14.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:06:14.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:06:14.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:06:14.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:06:14.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.234.126:34333,127.7.234.125:33623,127.7.234.124:38601
02:06:14.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:06:14.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35625/jwks.json
02:06:14.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
02:06:14.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:14.940639 8134 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:06:14.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:14.941202 8134 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:06:14.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:14.941792 8134 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:06:14.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:14.973208 8134 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
02:06:14.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:14.973456 8134 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
02:06:14.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:14.973726 8134 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:06:14.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:14.973922 8134 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
02:06:14.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:14.974115 8134 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
02:06:14.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:14.974300 8134 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
02:06:14.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:14.974499 8134 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:35625/jwks.json
02:06:15.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:15.013542 8134 master_runner.cc:386] Master server non-default flags:
02:06:15.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:06:15.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.234.84:33745
02:06:15.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:06:15.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:06:15.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/master-0/data
02:06:15.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/master-0/wal
02:06:15.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
02:06:15.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:06:15.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.234.126:34333,127.7.234.125:33623,127.7.234.124:38601
02:06:15.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/krb5kdc/kudu_127.7.234.126.keytab
02:06:15.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.7.234.126
02:06:15.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:06:15.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:06:15.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:06:15.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.234.126:34333
02:06:15.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:06:15.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:06:15.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:06:15.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35625/jwks.json
02:06:15.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:06:15.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
02:06:15.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
02:06:15.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:06:15.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:06:15.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/master-0/data/info.pb
02:06:15.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.234.126
02:06:15.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:06:15.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:06:15.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:06:15.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:06:15.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:06:15.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:06:15.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/master-0/logs
02:06:15.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:06:15.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:06:15.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:06:15.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:06:15.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
02:06:15.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision f0776864a65c19ebdd909053f3e624e8bcc25cee
02:06:15.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:06:15.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 28 Jun 2025 01:43:19 UTC on 5fd53c4cbb9d
02:06:15.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 6805
02:06:15.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:06:15.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:15.014832 8134 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:06:15.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:15.015872 8134 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:06:15.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:15.033231 8141 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:16.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:15.033284 8144 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:16.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:15.033437 8142 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:16.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:15.034287 8134 server_base.cc:1048] running on GCE node
02:06:16.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:16.262207 8134 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:06:16.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:16.265302 8134 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:06:16.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:16.266754 8134 hybrid_clock.cc:648] HybridClock initialized: now 1751076376266690 us; error 82 us; skew 500 ppm
02:06:16.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:16.267527 8134 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:06:16.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 28 02:06:16 dist-test-slave-lq1z krb5kdc[8119](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1751076376, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.234.126@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:06:16.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:16.273444 8134 init.cc:377] Logged in from keytab as kudu/127.7.234.126@KRBTEST.COM (short username kudu)
02:06:16.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:16.279590 8134 webserver.cc:469] Webserver started at http://127.7.234.126:37231/ using document root <none> and password file <none>
02:06:16.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:16.280604 8134 fs_manager.cc:362] Metadata directory not provided
02:06:16.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:16.280834 8134 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:06:16.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:16.281348 8134 server_base.cc:896] This appears to be a new deployment of Kudu; creating new FS layout
02:06:16.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:16.286116 8134 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/master-0/data/instance:
02:06:16.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6a385942aca04c84b16cd85c5407bae0"
02:06:16.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-28 02:06:16 on dist-test-slave-lq1z"
02:06:16.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:16.287273 8134 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/master-0/wal/instance:
02:06:16.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6a385942aca04c84b16cd85c5407bae0"
02:06:16.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-28 02:06:16 on dist-test-slave-lq1z"
02:06:16.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:16.294687 8134 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.009s sys 0.000s
02:06:16.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:16.300619 8152 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:06:16.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:16.301630 8134 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.002s sys 0.002s
02:06:16.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:16.301978 8134 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/master-0/data,/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/master-0/wal
02:06:16.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6a385942aca04c84b16cd85c5407bae0"
02:06:16.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-28 02:06:16 on dist-test-slave-lq1z"
02:06:16.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:16.302321 8134 fs_report.cc:389] FS layout report
02:06:16.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:06:16.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/master-0/wal
02:06:16.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/master-0/wal
02:06:16.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/master-0/data/data
02:06:16.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:06:16.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:06:16.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:06:16.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:06:16.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:06:16.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:06:16.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:06:16.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:06:16.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:06:16.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:06:16.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:06:16.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:16.357812 8134 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:06:16.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:16.363477 8134 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:06:16.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:16.363965 8134 kserver.cc:163] Server-wide thread pool size limit: 3276
02:06:16.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:16.443867 8134 rpc_server.cc:307] RPC server started. Bound to: 127.7.234.126:34333
02:06:16.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:16.443938 8203 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.7.234.126:34333 every 8 connection(s)
02:06:16.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:16.446520 8134 server_base.cc:1180] Dumped server information to /tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/master-0/data/info.pb
02:06:16.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:16.449770 8105 external_mini_cluster.cc:1413] Started /tmp/dist-test-task8JcgIg/build/tsan/bin/kudu as pid 8134
02:06:16.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:16.450290 8105 external_mini_cluster.cc:1427] Reading /tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/master-0/wal/instance
02:06:16.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:16.452762 8204 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
02:06:16.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:16.467868 8204 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.234.126" port: 34333 } has no permanent_uuid. Determining permanent_uuid...
02:06:16.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.7.234.125@KRBTEST.COM; defaulting to no policy
02:06:16.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:06:16.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.7.234.125@KRBTEST.COM" created.
02:06:16.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 28 02:06:16 dist-test-slave-lq1z krb5kdc[8119](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1751076376, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.234.126@KRBTEST.COM for kudu/127.7.234.126@KRBTEST.COM
02:06:16.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:06:16.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.7.234.125 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/krb5kdc/kudu_127.7.234.125.keytab.
02:06:16.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.7.234.125 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/krb5kdc/kudu_127.7.234.125.keytab.
02:06:16.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:16.501772 8105 external_mini_cluster.cc:1351] Running /tmp/dist-test-task8JcgIg/build/tsan/bin/kudu
02:06:16.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task8JcgIg/build/tsan/bin/kudu
02:06:16.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/master-1/wal
02:06:16.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/master-1/data
02:06:16.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:06:16.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:06:16.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:06:16.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:06:16.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:06:16.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:06:16.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/master-1/logs
02:06:16.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/master-1/data/info.pb
02:06:16.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:06:16.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:06:16.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:06:16.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:06:16.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:06:16.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:06:16.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:06:16.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:06:16.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:06:16.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:06:16.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:06:16.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:06:16.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.234.125:33623
02:06:16.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.234.125
02:06:16.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:06:16.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/krb5kdc/kudu_127.7.234.125.keytab
02:06:16.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.7.234.125
02:06:16.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:06:16.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
02:06:16.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
02:06:16.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.234.84:33745
02:06:16.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:06:16.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:06:16.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:06:16.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:06:16.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.234.126:34333,127.7.234.125:33623,127.7.234.124:38601
02:06:16.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:06:16.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35625/jwks.json
02:06:16.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
02:06:16.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:16.509104 8204 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.234.125" port: 33623 } has no permanent_uuid. Determining permanent_uuid...
02:06:16.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:16.512275 8153 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.234.125:33623: connect: Connection refused (error 111)
02:06:16.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:16.517426 8204 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.7.234.125:33623: Network error: Client connection negotiation failed: client connection to 127.7.234.125:33623: connect: Connection refused (error 111)
02:06:16.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:16.566963 8204 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.234.125" port: 33623 } attempt: 1
02:06:16.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:16.570551 8204 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.7.234.125:33623: Network error: Client connection negotiation failed: client connection to 127.7.234.125:33623: connect: Connection refused (error 111)
02:06:16.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:16.638922 8204 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.234.125" port: 33623 } attempt: 2
02:06:16.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:16.643285 8204 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.7.234.125:33623: Network error: Client connection negotiation failed: client connection to 127.7.234.125:33623: connect: Connection refused (error 111)
02:06:16.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:16.734689 8204 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.234.125" port: 33623 } attempt: 3
02:06:16.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:16.739434 8204 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.7.234.125:33623: Network error: Client connection negotiation failed: client connection to 127.7.234.125:33623: connect: Connection refused (error 111)
02:06:16.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:16.824792 8210 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:06:16.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:16.825441 8210 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:06:16.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:16.826134 8210 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:06:16.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:16.857973 8210 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
02:06:16.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:16.858258 8210 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
02:06:16.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:16.858704 8210 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:06:16.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:16.859019 8210 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
02:06:16.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:16.859344 8210 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
02:06:16.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:16.859645 8210 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
02:06:16.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:16.859858 8210 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:35625/jwks.json
02:06:16.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:16.882889 8204 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.234.125" port: 33623 } attempt: 4
02:06:16.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:16.886718 8204 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.7.234.125:33623: Network error: Client connection negotiation failed: client connection to 127.7.234.125:33623: connect: Connection refused (error 111)
02:06:16.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:16.900413 8210 master_runner.cc:386] Master server non-default flags:
02:06:16.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:06:16.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.234.84:33745
02:06:16.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:06:16.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:06:16.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/master-1/data
02:06:16.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/master-1/wal
02:06:16.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
02:06:16.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:06:16.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.234.126:34333,127.7.234.125:33623,127.7.234.124:38601
02:06:16.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/krb5kdc/kudu_127.7.234.125.keytab
02:06:16.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.7.234.125
02:06:16.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:06:16.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:06:16.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:06:16.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.234.125:33623
02:06:16.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:06:16.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:06:16.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:06:16.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35625/jwks.json
02:06:16.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:06:16.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
02:06:16.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
02:06:16.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:06:16.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:06:16.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/master-1/data/info.pb
02:06:16.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.234.125
02:06:16.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:06:16.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:06:16.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:06:16.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:06:16.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:06:16.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:06:16.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/master-1/logs
02:06:16.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:06:16.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:06:16.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:06:16.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:06:16.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
02:06:16.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision f0776864a65c19ebdd909053f3e624e8bcc25cee
02:06:16.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:06:16.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 28 Jun 2025 01:43:19 UTC on 5fd53c4cbb9d
02:06:16.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 6805
02:06:16.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:06:16.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:16.902067 8210 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:06:16.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:16.903479 8210 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:06:16.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:16.923880 8217 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:17.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:17.186053 8204 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.234.125" port: 33623 } attempt: 5
02:06:17.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:17.189536 8204 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.7.234.125:33623: Network error: Client connection negotiation failed: client connection to 127.7.234.125:33623: connect: Connection refused (error 111)
02:06:17.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:17.736892 8204 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.234.125" port: 33623 } attempt: 6
02:06:17.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:17.745887 8204 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.7.234.125:33623: Network error: Client connection negotiation failed: client connection to 127.7.234.125:33623: connect: Connection refused (error 111)
02:06:18.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:16.924506 8218 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:18.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:16.927201 8210 server_base.cc:1048] running on GCE node
02:06:18.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:16.923894 8220 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:18.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:18.148072 8210 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:06:18.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:18.151239 8210 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:06:18.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:18.152755 8210 hybrid_clock.cc:648] HybridClock initialized: now 1751076378152668 us; error 84 us; skew 500 ppm
02:06:18.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:18.153553 8210 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:06:18.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 28 02:06:18 dist-test-slave-lq1z krb5kdc[8119](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1751076378, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.234.125@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:06:18.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:18.163522 8210 init.cc:377] Logged in from keytab as kudu/127.7.234.125@KRBTEST.COM (short username kudu)
02:06:18.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:18.168753 8210 webserver.cc:469] Webserver started at http://127.7.234.125:38081/ using document root <none> and password file <none>
02:06:18.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:18.169862 8210 fs_manager.cc:362] Metadata directory not provided
02:06:18.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:18.170130 8210 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:06:18.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:18.170552 8210 server_base.cc:896] This appears to be a new deployment of Kudu; creating new FS layout
02:06:18.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:18.174908 8210 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/master-1/data/instance:
02:06:18.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "58a5c5945c8a4f35b49aa5dcbed1c006"
02:06:18.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-28 02:06:18 on dist-test-slave-lq1z"
02:06:18.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:18.176012 8210 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/master-1/wal/instance:
02:06:18.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "58a5c5945c8a4f35b49aa5dcbed1c006"
02:06:18.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-28 02:06:18 on dist-test-slave-lq1z"
02:06:18.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:18.183492 8210 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.006s sys 0.000s
02:06:18.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:18.188863 8231 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:06:18.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:18.189848 8210 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.000s sys 0.004s
02:06:18.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:18.190181 8210 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/master-1/data,/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/master-1/wal
02:06:18.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "58a5c5945c8a4f35b49aa5dcbed1c006"
02:06:18.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-28 02:06:18 on dist-test-slave-lq1z"
02:06:18.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:18.190515 8210 fs_report.cc:389] FS layout report
02:06:18.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:06:18.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/master-1/wal
02:06:18.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/master-1/wal
02:06:18.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/master-1/data/data
02:06:18.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:06:18.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:06:18.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:06:18.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:06:18.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:06:18.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:06:18.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:06:18.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:06:18.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:06:18.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:06:18.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:06:18.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:18.246217 8210 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:06:18.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:18.251972 8210 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:06:18.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:18.252595 8210 kserver.cc:163] Server-wide thread pool size limit: 3276
02:06:18.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:18.333154 8210 rpc_server.cc:307] RPC server started. Bound to: 127.7.234.125:33623
02:06:18.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:18.333230 8283 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.7.234.125:33623 every 8 connection(s)
02:06:18.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:18.335991 8210 server_base.cc:1180] Dumped server information to /tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/master-1/data/info.pb
02:06:18.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:18.341823 8284 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
02:06:18.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:18.342264 8105 external_mini_cluster.cc:1413] Started /tmp/dist-test-task8JcgIg/build/tsan/bin/kudu as pid 8210
02:06:18.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:18.342739 8105 external_mini_cluster.cc:1427] Reading /tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/master-1/wal/instance
02:06:18.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:18.357342 8284 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.234.126" port: 34333 } has no permanent_uuid. Determining permanent_uuid...
02:06:18.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.7.234.124@KRBTEST.COM; defaulting to no policy
02:06:18.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:06:18.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.7.234.124@KRBTEST.COM" created.
02:06:18.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 28 02:06:18 dist-test-slave-lq1z krb5kdc[8119](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1751076378, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.234.125@KRBTEST.COM for kudu/127.7.234.126@KRBTEST.COM
02:06:18.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:18.401000 8284 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.234.125" port: 33623 } has no permanent_uuid. Determining permanent_uuid...
02:06:18.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:06:18.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.7.234.124 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/krb5kdc/kudu_127.7.234.124.keytab.
02:06:18.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.7.234.124 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/krb5kdc/kudu_127.7.234.124.keytab.
02:06:18.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 28 02:06:18 dist-test-slave-lq1z krb5kdc[8119](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1751076378, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.234.125@KRBTEST.COM for kudu/127.7.234.125@KRBTEST.COM
02:06:18.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:18.416740 8105 external_mini_cluster.cc:1351] Running /tmp/dist-test-task8JcgIg/build/tsan/bin/kudu
02:06:18.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task8JcgIg/build/tsan/bin/kudu
02:06:18.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/master-2/wal
02:06:18.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/master-2/data
02:06:18.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:06:18.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:06:18.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:06:18.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:06:18.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:06:18.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:06:18.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/master-2/logs
02:06:18.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/master-2/data/info.pb
02:06:18.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:06:18.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:06:18.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:06:18.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:06:18.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:06:18.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:06:18.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:06:18.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:06:18.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:06:18.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:06:18.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:06:18.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:06:18.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.234.124:38601
02:06:18.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.234.124
02:06:18.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:06:18.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/krb5kdc/kudu_127.7.234.124.keytab
02:06:18.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.7.234.124
02:06:18.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:06:18.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
02:06:18.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
02:06:18.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.234.84:33745
02:06:18.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:06:18.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:06:18.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:06:18.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:06:18.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.234.126:34333,127.7.234.125:33623,127.7.234.124:38601
02:06:18.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:06:18.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35625/jwks.json
02:06:18.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
02:06:18.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:18.428393 8284 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.234.124" port: 38601 } has no permanent_uuid. Determining permanent_uuid...
02:06:18.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:18.431077 8234 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.234.124:38601: connect: Connection refused (error 111)
02:06:18.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:18.435648 8284 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.7.234.124:38601: Network error: Client connection negotiation failed: client connection to 127.7.234.124:38601: connect: Connection refused (error 111)
02:06:18.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:18.485278 8284 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.234.124" port: 38601 } attempt: 1
02:06:18.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:18.489022 8284 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.7.234.124:38601: Network error: Client connection negotiation failed: client connection to 127.7.234.124:38601: connect: Connection refused (error 111)
02:06:18.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:18.557864 8284 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.234.124" port: 38601 } attempt: 2
02:06:18.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:18.561530 8284 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.7.234.124:38601: Network error: Client connection negotiation failed: client connection to 127.7.234.124:38601: connect: Connection refused (error 111)
02:06:18.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:18.652951 8284 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.234.124" port: 38601 } attempt: 3
02:06:18.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:18.656560 8284 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.7.234.124:38601: Network error: Client connection negotiation failed: client connection to 127.7.234.124:38601: connect: Connection refused (error 111)
02:06:18.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:18.756073 8291 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:06:18.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:18.756704 8291 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:06:18.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:18.757293 8291 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:06:18.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:18.789243 8291 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
02:06:18.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:18.789463 8291 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
02:06:18.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:18.789724 8291 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:06:18.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:18.789914 8291 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
02:06:18.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:18.790102 8291 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
02:06:18.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:18.790278 8291 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
02:06:18.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:18.790412 8291 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:35625/jwks.json
02:06:18.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:18.799947 8284 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.234.124" port: 38601 } attempt: 4
02:06:18.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:18.803642 8284 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.7.234.124:38601: Network error: Client connection negotiation failed: client connection to 127.7.234.124:38601: connect: Connection refused (error 111)
02:06:18.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:18.806309 8204 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.234.125" port: 33623 } attempt: 7
02:06:18.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 28 02:06:18 dist-test-slave-lq1z krb5kdc[8119](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1751076376, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.234.126@KRBTEST.COM for kudu/127.7.234.125@KRBTEST.COM
02:06:18.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:18.828770 8204 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.234.124" port: 38601 } has no permanent_uuid. Determining permanent_uuid...
02:06:18.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:18.834611 8204 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.7.234.124:38601: Network error: Client connection negotiation failed: client connection to 127.7.234.124:38601: connect: Connection refused (error 111)
02:06:18.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:18.836905 8291 master_runner.cc:386] Master server non-default flags:
02:06:18.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:06:18.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.234.84:33745
02:06:18.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:06:18.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:06:18.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/master-2/data
02:06:18.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/master-2/wal
02:06:18.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
02:06:18.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:06:18.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.234.126:34333,127.7.234.125:33623,127.7.234.124:38601
02:06:18.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/krb5kdc/kudu_127.7.234.124.keytab
02:06:18.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.7.234.124
02:06:18.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:06:18.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:06:18.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:06:18.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.234.124:38601
02:06:18.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:06:18.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:06:18.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:06:18.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35625/jwks.json
02:06:18.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:06:18.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
02:06:18.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
02:06:18.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:06:18.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:06:18.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/master-2/data/info.pb
02:06:18.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.234.124
02:06:18.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:06:18.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:06:18.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:06:18.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:06:18.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:06:18.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:06:18.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/master-2/logs
02:06:18.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:06:18.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:06:18.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:06:18.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:06:18.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
02:06:18.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision f0776864a65c19ebdd909053f3e624e8bcc25cee
02:06:18.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:06:18.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 28 Jun 2025 01:43:19 UTC on 5fd53c4cbb9d
02:06:18.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 6805
02:06:18.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:06:18.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:18.838604 8291 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:06:18.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:18.839883 8291 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:06:18.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:18.856235 8302 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:18.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:18.892942 8204 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.234.124" port: 38601 } attempt: 1
02:06:18.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:18.896607 8204 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.7.234.124:38601: Network error: Client connection negotiation failed: client connection to 127.7.234.124:38601: connect: Connection refused (error 111)
02:06:18.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:18.977993 8204 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.234.124" port: 38601 } attempt: 2
02:06:18.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:18.983167 8204 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.7.234.124:38601: Network error: Client connection negotiation failed: client connection to 127.7.234.124:38601: connect: Connection refused (error 111)
02:06:19.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:19.068500 8204 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.234.124" port: 38601 } attempt: 3
02:06:19.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:19.072801 8204 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.7.234.124:38601: Network error: Client connection negotiation failed: client connection to 127.7.234.124:38601: connect: Connection refused (error 111)
02:06:19.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:19.103053 8284 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.234.124" port: 38601 } attempt: 5
02:06:19.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:19.106774 8284 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.7.234.124:38601: Network error: Client connection negotiation failed: client connection to 127.7.234.124:38601: connect: Connection refused (error 111)
02:06:19.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:19.213145 8204 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.234.124" port: 38601 } attempt: 4
02:06:19.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:19.216732 8204 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.7.234.124:38601: Network error: Client connection negotiation failed: client connection to 127.7.234.124:38601: connect: Connection refused (error 111)
02:06:19.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:19.500344 8204 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.234.124" port: 38601 } attempt: 5
02:06:19.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:19.504550 8204 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.7.234.124:38601: Network error: Client connection negotiation failed: client connection to 127.7.234.124:38601: connect: Connection refused (error 111)
02:06:19.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:19.654151 8284 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.234.124" port: 38601 } attempt: 6
02:06:19.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:19.662359 8284 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.7.234.124:38601: Network error: Client connection negotiation failed: client connection to 127.7.234.124:38601: connect: Connection refused (error 111)
02:06:20.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:20.057127 8204 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.234.124" port: 38601 } attempt: 6
02:06:20.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:20.063511 8204 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.7.234.124:38601: Network error: Client connection negotiation failed: client connection to 127.7.234.124:38601: connect: Connection refused (error 111)
02:06:20.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:18.856313 8304 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:20.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:18.857010 8301 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:20.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:18.860126 8291 server_base.cc:1048] running on GCE node
02:06:20.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:20.088269 8291 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:06:20.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:20.090736 8291 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:06:20.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:20.092096 8291 hybrid_clock.cc:648] HybridClock initialized: now 1751076380092054 us; error 57 us; skew 500 ppm
02:06:20.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:20.092842 8291 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:06:20.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 28 02:06:20 dist-test-slave-lq1z krb5kdc[8119](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1751076380, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.234.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:06:20.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:20.098507 8291 init.cc:377] Logged in from keytab as kudu/127.7.234.124@KRBTEST.COM (short username kudu)
02:06:20.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:20.103986 8291 webserver.cc:469] Webserver started at http://127.7.234.124:40485/ using document root <none> and password file <none>
02:06:20.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:20.104959 8291 fs_manager.cc:362] Metadata directory not provided
02:06:20.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:20.105172 8291 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:06:20.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:20.105677 8291 server_base.cc:896] This appears to be a new deployment of Kudu; creating new FS layout
02:06:20.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:20.110504 8291 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/master-2/data/instance:
02:06:20.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "16434748810647be9737fc5cf81d3164"
02:06:20.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-28 02:06:20 on dist-test-slave-lq1z"
02:06:20.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:20.111611 8291 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/master-2/wal/instance:
02:06:20.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "16434748810647be9737fc5cf81d3164"
02:06:20.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-28 02:06:20 on dist-test-slave-lq1z"
02:06:20.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:20.119349 8291 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.005s sys 0.001s
02:06:20.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:20.126711 8318 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:06:20.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:20.127748 8291 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.005s sys 0.001s
02:06:20.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:20.128027 8291 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/master-2/data,/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/master-2/wal
02:06:20.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "16434748810647be9737fc5cf81d3164"
02:06:20.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-28 02:06:20 on dist-test-slave-lq1z"
02:06:20.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:20.128372 8291 fs_report.cc:389] FS layout report
02:06:20.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:06:20.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/master-2/wal
02:06:20.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/master-2/wal
02:06:20.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/master-2/data/data
02:06:20.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:06:20.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:06:20.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:06:20.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:06:20.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:06:20.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:06:20.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:06:20.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:06:20.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:06:20.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:06:20.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:06:20.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:20.191098 8291 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:06:20.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:20.196692 8291 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:06:20.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:20.197090 8291 kserver.cc:163] Server-wide thread pool size limit: 3276
02:06:20.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:20.276235 8291 rpc_server.cc:307] RPC server started. Bound to: 127.7.234.124:38601
02:06:20.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:20.276357 8369 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.7.234.124:38601 every 8 connection(s)
02:06:20.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:20.279112 8291 server_base.cc:1180] Dumped server information to /tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/master-2/data/info.pb
02:06:20.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:20.284425 8370 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
02:06:20.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:20.288542 8105 external_mini_cluster.cc:1413] Started /tmp/dist-test-task8JcgIg/build/tsan/bin/kudu as pid 8291
02:06:20.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:20.288879 8105 external_mini_cluster.cc:1427] Reading /tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/master-2/wal/instance
02:06:20.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:20.298484 8370 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.234.126" port: 34333 } has no permanent_uuid. Determining permanent_uuid...
02:06:20.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 28 02:06:20 dist-test-slave-lq1z krb5kdc[8119](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1751076374, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.7.234.126@KRBTEST.COM
02:06:20.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 28 02:06:20 dist-test-slave-lq1z krb5kdc[8119](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1751076380, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.234.124@KRBTEST.COM for kudu/127.7.234.126@KRBTEST.COM
02:06:20.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:20.347307 8370 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.234.125" port: 33623 } has no permanent_uuid. Determining permanent_uuid...
02:06:20.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 28 02:06:20 dist-test-slave-lq1z krb5kdc[8119](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1751076374, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.7.234.125@KRBTEST.COM
02:06:20.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 28 02:06:20 dist-test-slave-lq1z krb5kdc[8119](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1751076380, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.234.124@KRBTEST.COM for kudu/127.7.234.125@KRBTEST.COM
02:06:20.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:20.370208 8370 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.234.124" port: 38601 } has no permanent_uuid. Determining permanent_uuid...
02:06:20.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 28 02:06:20 dist-test-slave-lq1z krb5kdc[8119](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1751076374, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.7.234.124@KRBTEST.COM
02:06:20.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 28 02:06:20 dist-test-slave-lq1z krb5kdc[8119](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1751076380, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.234.124@KRBTEST.COM for kudu/127.7.234.124@KRBTEST.COM
02:06:20.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:20.399036 8370 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 16434748810647be9737fc5cf81d3164: Bootstrap starting.
02:06:20.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:20.405685 8370 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 16434748810647be9737fc5cf81d3164: Neither blocks nor log segments found. Creating new log.
02:06:20.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:20.407400 8370 log.cc:826] T 00000000000000000000000000000000 P 16434748810647be9737fc5cf81d3164: Log is configured to *not* fsync() on all Append() calls
02:06:20.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:20.413581 8370 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 16434748810647be9737fc5cf81d3164: No bootstrap required, opened a new log
02:06:20.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:20.433269 8370 raft_consensus.cc:357] T 00000000000000000000000000000000 P 16434748810647be9737fc5cf81d3164 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a385942aca04c84b16cd85c5407bae0" member_type: VOTER last_known_addr { host: "127.7.234.126" port: 34333 } } peers { permanent_uuid: "58a5c5945c8a4f35b49aa5dcbed1c006" member_type: VOTER last_known_addr { host: "127.7.234.125" port: 33623 } } peers { permanent_uuid: "16434748810647be9737fc5cf81d3164" member_type: VOTER last_known_addr { host: "127.7.234.124" port: 38601 } }
02:06:20.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:20.433908 8370 raft_consensus.cc:383] T 00000000000000000000000000000000 P 16434748810647be9737fc5cf81d3164 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:06:20.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:20.434093 8370 raft_consensus.cc:738] T 00000000000000000000000000000000 P 16434748810647be9737fc5cf81d3164 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 16434748810647be9737fc5cf81d3164, State: Initialized, Role: FOLLOWER
02:06:20.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:20.434736 8370 consensus_queue.cc:260] T 00000000000000000000000000000000 P 16434748810647be9737fc5cf81d3164 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a385942aca04c84b16cd85c5407bae0" member_type: VOTER last_known_addr { host: "127.7.234.126" port: 34333 } } peers { permanent_uuid: "58a5c5945c8a4f35b49aa5dcbed1c006" member_type: VOTER last_known_addr { host: "127.7.234.125" port: 33623 } } peers { permanent_uuid: "16434748810647be9737fc5cf81d3164" member_type: VOTER last_known_addr { host: "127.7.234.124" port: 38601 } }
02:06:20.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:20.438261 8381 sys_catalog.cc:455] T 00000000000000000000000000000000 P 16434748810647be9737fc5cf81d3164 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a385942aca04c84b16cd85c5407bae0" member_type: VOTER last_known_addr { host: "127.7.234.126" port: 34333 } } peers { permanent_uuid: "58a5c5945c8a4f35b49aa5dcbed1c006" member_type: VOTER last_known_addr { host: "127.7.234.125" port: 33623 } } peers { permanent_uuid: "16434748810647be9737fc5cf81d3164" member_type: VOTER last_known_addr { host: "127.7.234.124" port: 38601 } } }
02:06:20.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:20.439044 8381 sys_catalog.cc:458] T 00000000000000000000000000000000 P 16434748810647be9737fc5cf81d3164 [sys.catalog]: This master's current role is: FOLLOWER
02:06:20.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:20.440259 8370 sys_catalog.cc:564] T 00000000000000000000000000000000 P 16434748810647be9737fc5cf81d3164 [sys.catalog]: configured and running, proceeding with master startup.
02:06:20.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:20.452348 8324 tablet.cc:2378] T 00000000000000000000000000000000 P 16434748810647be9737fc5cf81d3164: Can't schedule compaction. Clean time has not been advanced past its initial value.
02:06:20.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:20.484158 8393 catalog_manager.cc:1560] T 00000000000000000000000000000000 P 16434748810647be9737fc5cf81d3164: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:06:20.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:20.484530 8393 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:06:20.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:20.478947 8291 master_runner.cc:186] Error getting master registration for 127.7.234.126:34333: OK, instance_id { permanent_uuid: "6a385942aca04c84b16cd85c5407bae0" instance_seqno: 1751076376422769 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
02:06:20.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:20.501765 8291 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
02:06:20.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:20.722679 8284 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.234.124" port: 38601 } attempt: 7
02:06:20.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 28 02:06:20 dist-test-slave-lq1z krb5kdc[8119](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1751076378, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.234.125@KRBTEST.COM for kudu/127.7.234.124@KRBTEST.COM
02:06:20.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:20.759343 8284 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 58a5c5945c8a4f35b49aa5dcbed1c006: Bootstrap starting.
02:06:20.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:20.765090 8284 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 58a5c5945c8a4f35b49aa5dcbed1c006: Neither blocks nor log segments found. Creating new log.
02:06:20.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:20.766758 8284 log.cc:826] T 00000000000000000000000000000000 P 58a5c5945c8a4f35b49aa5dcbed1c006: Log is configured to *not* fsync() on all Append() calls
02:06:20.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:20.771826 8284 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 58a5c5945c8a4f35b49aa5dcbed1c006: No bootstrap required, opened a new log
02:06:20.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:20.791345 8284 raft_consensus.cc:357] T 00000000000000000000000000000000 P 58a5c5945c8a4f35b49aa5dcbed1c006 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a385942aca04c84b16cd85c5407bae0" member_type: VOTER last_known_addr { host: "127.7.234.126" port: 34333 } } peers { permanent_uuid: "58a5c5945c8a4f35b49aa5dcbed1c006" member_type: VOTER last_known_addr { host: "127.7.234.125" port: 33623 } } peers { permanent_uuid: "16434748810647be9737fc5cf81d3164" member_type: VOTER last_known_addr { host: "127.7.234.124" port: 38601 } }
02:06:20.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:20.791981 8284 raft_consensus.cc:383] T 00000000000000000000000000000000 P 58a5c5945c8a4f35b49aa5dcbed1c006 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:06:20.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:20.792272 8284 raft_consensus.cc:738] T 00000000000000000000000000000000 P 58a5c5945c8a4f35b49aa5dcbed1c006 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 58a5c5945c8a4f35b49aa5dcbed1c006, State: Initialized, Role: FOLLOWER
02:06:20.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:20.793027 8284 consensus_queue.cc:260] T 00000000000000000000000000000000 P 58a5c5945c8a4f35b49aa5dcbed1c006 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a385942aca04c84b16cd85c5407bae0" member_type: VOTER last_known_addr { host: "127.7.234.126" port: 34333 } } peers { permanent_uuid: "58a5c5945c8a4f35b49aa5dcbed1c006" member_type: VOTER last_known_addr { host: "127.7.234.125" port: 33623 } } peers { permanent_uuid: "16434748810647be9737fc5cf81d3164" member_type: VOTER last_known_addr { host: "127.7.234.124" port: 38601 } }
02:06:20.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:20.795867 8397 sys_catalog.cc:455] T 00000000000000000000000000000000 P 58a5c5945c8a4f35b49aa5dcbed1c006 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a385942aca04c84b16cd85c5407bae0" member_type: VOTER last_known_addr { host: "127.7.234.126" port: 34333 } } peers { permanent_uuid: "58a5c5945c8a4f35b49aa5dcbed1c006" member_type: VOTER last_known_addr { host: "127.7.234.125" port: 33623 } } peers { permanent_uuid: "16434748810647be9737fc5cf81d3164" member_type: VOTER last_known_addr { host: "127.7.234.124" port: 38601 } } }
02:06:20.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:20.796841 8397 sys_catalog.cc:458] T 00000000000000000000000000000000 P 58a5c5945c8a4f35b49aa5dcbed1c006 [sys.catalog]: This master's current role is: FOLLOWER
02:06:20.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:20.798214 8284 sys_catalog.cc:564] T 00000000000000000000000000000000 P 58a5c5945c8a4f35b49aa5dcbed1c006 [sys.catalog]: configured and running, proceeding with master startup.
02:06:20.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:20.817700 8381 raft_consensus.cc:491] T 00000000000000000000000000000000 P 16434748810647be9737fc5cf81d3164 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:06:20.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:20.818279 8381 raft_consensus.cc:513] T 00000000000000000000000000000000 P 16434748810647be9737fc5cf81d3164 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a385942aca04c84b16cd85c5407bae0" member_type: VOTER last_known_addr { host: "127.7.234.126" port: 34333 } } peers { permanent_uuid: "58a5c5945c8a4f35b49aa5dcbed1c006" member_type: VOTER last_known_addr { host: "127.7.234.125" port: 33623 } } peers { permanent_uuid: "16434748810647be9737fc5cf81d3164" member_type: VOTER last_known_addr { host: "127.7.234.124" port: 38601 } }
02:06:20.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:20.822520 8381 leader_election.cc:290] T 00000000000000000000000000000000 P 16434748810647be9737fc5cf81d3164 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6a385942aca04c84b16cd85c5407bae0 (127.7.234.126:34333), 58a5c5945c8a4f35b49aa5dcbed1c006 (127.7.234.125:33623)
02:06:20.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:20.822340 8259 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "16434748810647be9737fc5cf81d3164" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "58a5c5945c8a4f35b49aa5dcbed1c006" is_pre_election: true
02:06:20.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:20.823390 8259 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 58a5c5945c8a4f35b49aa5dcbed1c006 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 16434748810647be9737fc5cf81d3164 in term 0.
02:06:20.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:20.826634 8319 leader_election.cc:304] T 00000000000000000000000000000000 P 16434748810647be9737fc5cf81d3164 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 16434748810647be9737fc5cf81d3164, 58a5c5945c8a4f35b49aa5dcbed1c006; no voters:
02:06:20.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:20.827842 8381 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 16434748810647be9737fc5cf81d3164 [term 0 FOLLOWER]: Leader pre-election won for term 1
02:06:20.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:20.828195 8381 raft_consensus.cc:491] T 00000000000000000000000000000000 P 16434748810647be9737fc5cf81d3164 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
02:06:20.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:20.828497 8381 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 16434748810647be9737fc5cf81d3164 [term 0 FOLLOWER]: Advancing to term 1
02:06:20.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:20.835685 8381 raft_consensus.cc:513] T 00000000000000000000000000000000 P 16434748810647be9737fc5cf81d3164 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a385942aca04c84b16cd85c5407bae0" member_type: VOTER last_known_addr { host: "127.7.234.126" port: 34333 } } peers { permanent_uuid: "58a5c5945c8a4f35b49aa5dcbed1c006" member_type: VOTER last_known_addr { host: "127.7.234.125" port: 33623 } } peers { permanent_uuid: "16434748810647be9737fc5cf81d3164" member_type: VOTER last_known_addr { host: "127.7.234.124" port: 38601 } }
02:06:20.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:20.845129 8381 leader_election.cc:290] T 00000000000000000000000000000000 P 16434748810647be9737fc5cf81d3164 [CANDIDATE]: Term 1 election: Requested vote from peers 6a385942aca04c84b16cd85c5407bae0 (127.7.234.126:34333), 58a5c5945c8a4f35b49aa5dcbed1c006 (127.7.234.125:33623)
02:06:20.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:20.845341 8259 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "16434748810647be9737fc5cf81d3164" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "58a5c5945c8a4f35b49aa5dcbed1c006"
02:06:20.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:20.845912 8259 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 58a5c5945c8a4f35b49aa5dcbed1c006 [term 0 FOLLOWER]: Advancing to term 1
02:06:20.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:20.857824 8259 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 58a5c5945c8a4f35b49aa5dcbed1c006 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 16434748810647be9737fc5cf81d3164 in term 1.
02:06:20.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:20.859663 8319 leader_election.cc:304] T 00000000000000000000000000000000 P 16434748810647be9737fc5cf81d3164 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 16434748810647be9737fc5cf81d3164, 58a5c5945c8a4f35b49aa5dcbed1c006; no voters:
02:06:20.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:20.860759 8381 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 16434748810647be9737fc5cf81d3164 [term 1 FOLLOWER]: Leader election won for term 1
02:06:20.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:20.828083 8210 master_runner.cc:186] Error getting master registration for 127.7.234.126:34333: OK, instance_id { permanent_uuid: "6a385942aca04c84b16cd85c5407bae0" instance_seqno: 1751076376422769 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
02:06:20.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:20.863971 8210 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
02:06:20.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:20.868443 8381 raft_consensus.cc:695] T 00000000000000000000000000000000 P 16434748810647be9737fc5cf81d3164 [term 1 LEADER]: Becoming Leader. State: Replica: 16434748810647be9737fc5cf81d3164, State: Running, Role: LEADER
02:06:20.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:20.869439 8381 consensus_queue.cc:237] T 00000000000000000000000000000000 P 16434748810647be9737fc5cf81d3164 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a385942aca04c84b16cd85c5407bae0" member_type: VOTER last_known_addr { host: "127.7.234.126" port: 34333 } } peers { permanent_uuid: "58a5c5945c8a4f35b49aa5dcbed1c006" member_type: VOTER last_known_addr { host: "127.7.234.125" port: 33623 } } peers { permanent_uuid: "16434748810647be9737fc5cf81d3164" member_type: VOTER last_known_addr { host: "127.7.234.124" port: 38601 } }
02:06:20.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:20.870877 8408 catalog_manager.cc:1560] T 00000000000000000000000000000000 P 58a5c5945c8a4f35b49aa5dcbed1c006: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:06:20.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:20.871189 8408 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:06:20.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:20.884485 8410 sys_catalog.cc:455] T 00000000000000000000000000000000 P 16434748810647be9737fc5cf81d3164 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 16434748810647be9737fc5cf81d3164. Latest consensus state: current_term: 1 leader_uuid: "16434748810647be9737fc5cf81d3164" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a385942aca04c84b16cd85c5407bae0" member_type: VOTER last_known_addr { host: "127.7.234.126" port: 34333 } } peers { permanent_uuid: "58a5c5945c8a4f35b49aa5dcbed1c006" member_type: VOTER last_known_addr { host: "127.7.234.125" port: 33623 } } peers { permanent_uuid: "16434748810647be9737fc5cf81d3164" member_type: VOTER last_known_addr { host: "127.7.234.124" port: 38601 } } }
02:06:20.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:20.885248 8410 sys_catalog.cc:458] T 00000000000000000000000000000000 P 16434748810647be9737fc5cf81d3164 [sys.catalog]: This master's current role is: LEADER
02:06:20.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:20.890214 8412 catalog_manager.cc:1477] Loading table and tablet metadata into memory...
02:06:20.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:20.897769 8412 catalog_manager.cc:1486] Initializing Kudu cluster ID...
02:06:20.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:20.920166 8259 raft_consensus.cc:1273] T 00000000000000000000000000000000 P 58a5c5945c8a4f35b49aa5dcbed1c006 [term 1 FOLLOWER]: Refusing update from remote peer 16434748810647be9737fc5cf81d3164: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
02:06:20.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:20.921850 8410 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 16434748810647be9737fc5cf81d3164 [LEADER]: Connected to new peer: Peer: permanent_uuid: "58a5c5945c8a4f35b49aa5dcbed1c006" member_type: VOTER last_known_addr { host: "127.7.234.125" port: 33623 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
02:06:20.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:20.942763 8397 sys_catalog.cc:455] T 00000000000000000000000000000000 P 58a5c5945c8a4f35b49aa5dcbed1c006 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 16434748810647be9737fc5cf81d3164. Latest consensus state: current_term: 1 leader_uuid: "16434748810647be9737fc5cf81d3164" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a385942aca04c84b16cd85c5407bae0" member_type: VOTER last_known_addr { host: "127.7.234.126" port: 34333 } } peers { permanent_uuid: "58a5c5945c8a4f35b49aa5dcbed1c006" member_type: VOTER last_known_addr { host: "127.7.234.125" port: 33623 } } peers { permanent_uuid: "16434748810647be9737fc5cf81d3164" member_type: VOTER last_known_addr { host: "127.7.234.124" port: 38601 } } }
02:06:20.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:20.943493 8397 sys_catalog.cc:458] T 00000000000000000000000000000000 P 58a5c5945c8a4f35b49aa5dcbed1c006 [sys.catalog]: This master's current role is: FOLLOWER
02:06:20.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:20.949859 8413 mvcc.cc:204] Tried to move back new op lower bound from 7172408856228192256 to 7172408856051228672. Current Snapshot: MvccSnapshot[applied={T|T < 7172408856228192256}]
02:06:20.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:20.958456 8410 sys_catalog.cc:455] T 00000000000000000000000000000000 P 16434748810647be9737fc5cf81d3164 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "16434748810647be9737fc5cf81d3164" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a385942aca04c84b16cd85c5407bae0" member_type: VOTER last_known_addr { host: "127.7.234.126" port: 34333 } } peers { permanent_uuid: "58a5c5945c8a4f35b49aa5dcbed1c006" member_type: VOTER last_known_addr { host: "127.7.234.125" port: 33623 } } peers { permanent_uuid: "16434748810647be9737fc5cf81d3164" member_type: VOTER last_known_addr { host: "127.7.234.124" port: 38601 } } }
02:06:20.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:20.959175 8410 sys_catalog.cc:458] T 00000000000000000000000000000000 P 16434748810647be9737fc5cf81d3164 [sys.catalog]: This master's current role is: LEADER
02:06:20.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:20.961997 8397 sys_catalog.cc:455] T 00000000000000000000000000000000 P 58a5c5945c8a4f35b49aa5dcbed1c006 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "16434748810647be9737fc5cf81d3164" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a385942aca04c84b16cd85c5407bae0" member_type: VOTER last_known_addr { host: "127.7.234.126" port: 34333 } } peers { permanent_uuid: "58a5c5945c8a4f35b49aa5dcbed1c006" member_type: VOTER last_known_addr { host: "127.7.234.125" port: 33623 } } peers { permanent_uuid: "16434748810647be9737fc5cf81d3164" member_type: VOTER last_known_addr { host: "127.7.234.124" port: 38601 } } }
02:06:20.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:20.962663 8397 sys_catalog.cc:458] T 00000000000000000000000000000000 P 58a5c5945c8a4f35b49aa5dcbed1c006 [sys.catalog]: This master's current role is: FOLLOWER
02:06:20.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:20.976819 8412 catalog_manager.cc:1349] Generated new cluster ID: c5aad08e8e8643a6baa349b9d79cbd4e
02:06:20.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:20.977147 8412 catalog_manager.cc:1497] Initializing Kudu internal certificate authority...
02:06:21.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:21.028784 8412 catalog_manager.cc:1372] Generated new certificate authority record
02:06:21.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:21.034742 8412 catalog_manager.cc:1506] Loading token signing keys...
02:06:21.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:21.068492 8412 catalog_manager.cc:5955] T 00000000000000000000000000000000 P 16434748810647be9737fc5cf81d3164: Generated new TSK 0
02:06:21.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:21.071593 8412 catalog_manager.cc:1516] Initializing in-progress tserver states...
02:06:21.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:21.096987 8204 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.234.124" port: 38601 } attempt: 7
02:06:21.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 28 02:06:21 dist-test-slave-lq1z krb5kdc[8119](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1751076376, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.234.126@KRBTEST.COM for kudu/127.7.234.124@KRBTEST.COM
02:06:21.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.7.234.65@KRBTEST.COM; defaulting to no policy
02:06:21.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:06:21.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.7.234.65@KRBTEST.COM" created.
02:06:21.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:21.184374 8204 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6a385942aca04c84b16cd85c5407bae0: Bootstrap starting.
02:06:21.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:06:21.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.7.234.65 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/krb5kdc/kudu_127.7.234.65.keytab.
02:06:21.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.7.234.65 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/krb5kdc/kudu_127.7.234.65.keytab.
02:06:21.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:21.206625 8105 external_mini_cluster.cc:1351] Running /tmp/dist-test-task8JcgIg/build/tsan/bin/kudu
02:06:21.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task8JcgIg/build/tsan/bin/kudu
02:06:21.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/ts-0/wal
02:06:21.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/ts-0/data
02:06:21.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:06:21.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:06:21.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:06:21.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:06:21.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:06:21.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:06:21.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/ts-0/logs
02:06:21.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/ts-0/data/info.pb
02:06:21.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:06:21.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:06:21.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:06:21.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:06:21.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:06:21.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:06:21.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:06:21.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:06:21.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
02:06:21.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:06:21.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.234.65:0
02:06:21.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.7.234.65
02:06:21.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.234.65
02:06:21.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:06:21.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.7.234.126:34333,127.7.234.125:33623,127.7.234.124:38601
02:06:21.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/krb5kdc/kudu_127.7.234.65.keytab
02:06:21.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.7.234.65
02:06:21.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:06:21.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
02:06:21.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
02:06:21.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.234.84:33745
02:06:21.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:06:21.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:06:21.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {KRB5CCNAME=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
02:06:21.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:21.233965 8204 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 6a385942aca04c84b16cd85c5407bae0: Neither blocks nor log segments found. Creating new log.
02:06:21.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:21.254385 8204 log.cc:826] T 00000000000000000000000000000000 P 6a385942aca04c84b16cd85c5407bae0: Log is configured to *not* fsync() on all Append() calls
02:06:21.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:21.315476 8204 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6a385942aca04c84b16cd85c5407bae0: No bootstrap required, opened a new log
02:06:21.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:21.509667 8204 raft_consensus.cc:357] T 00000000000000000000000000000000 P 6a385942aca04c84b16cd85c5407bae0 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a385942aca04c84b16cd85c5407bae0" member_type: VOTER last_known_addr { host: "127.7.234.126" port: 34333 } } peers { permanent_uuid: "58a5c5945c8a4f35b49aa5dcbed1c006" member_type: VOTER last_known_addr { host: "127.7.234.125" port: 33623 } } peers { permanent_uuid: "16434748810647be9737fc5cf81d3164" member_type: VOTER last_known_addr { host: "127.7.234.124" port: 38601 } }
02:06:21.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:21.516471 8204 raft_consensus.cc:383] T 00000000000000000000000000000000 P 6a385942aca04c84b16cd85c5407bae0 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:06:21.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:21.516798 8204 raft_consensus.cc:738] T 00000000000000000000000000000000 P 6a385942aca04c84b16cd85c5407bae0 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a385942aca04c84b16cd85c5407bae0, State: Initialized, Role: FOLLOWER
02:06:21.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:21.517719 8204 consensus_queue.cc:260] T 00000000000000000000000000000000 P 6a385942aca04c84b16cd85c5407bae0 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a385942aca04c84b16cd85c5407bae0" member_type: VOTER last_known_addr { host: "127.7.234.126" port: 34333 } } peers { permanent_uuid: "58a5c5945c8a4f35b49aa5dcbed1c006" member_type: VOTER last_known_addr { host: "127.7.234.125" port: 33623 } } peers { permanent_uuid: "16434748810647be9737fc5cf81d3164" member_type: VOTER last_known_addr { host: "127.7.234.124" port: 38601 } }
02:06:21.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:21.530751 8423 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6a385942aca04c84b16cd85c5407bae0 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a385942aca04c84b16cd85c5407bae0" member_type: VOTER last_known_addr { host: "127.7.234.126" port: 34333 } } peers { permanent_uuid: "58a5c5945c8a4f35b49aa5dcbed1c006" member_type: VOTER last_known_addr { host: "127.7.234.125" port: 33623 } } peers { permanent_uuid: "16434748810647be9737fc5cf81d3164" member_type: VOTER last_known_addr { host: "127.7.234.124" port: 38601 } } }
02:06:21.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:21.535255 8423 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6a385942aca04c84b16cd85c5407bae0 [sys.catalog]: This master's current role is: FOLLOWER
02:06:21.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:21.538556 8179 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "16434748810647be9737fc5cf81d3164" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a385942aca04c84b16cd85c5407bae0" is_pre_election: true
02:06:21.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:21.539595 8179 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 6a385942aca04c84b16cd85c5407bae0 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 16434748810647be9737fc5cf81d3164 in term 0.
02:06:21.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:21.538575 8204 sys_catalog.cc:564] T 00000000000000000000000000000000 P 6a385942aca04c84b16cd85c5407bae0 [sys.catalog]: configured and running, proceeding with master startup.
02:06:21.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:21.540691 8178 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "16434748810647be9737fc5cf81d3164" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a385942aca04c84b16cd85c5407bae0"
02:06:21.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:21.541985 8178 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 6a385942aca04c84b16cd85c5407bae0 [term 0 FOLLOWER]: Advancing to term 1
02:06:21.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:21.550040 8178 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 6a385942aca04c84b16cd85c5407bae0 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 16434748810647be9737fc5cf81d3164 in term 1.
02:06:21.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:21.551645 8177 raft_consensus.cc:1273] T 00000000000000000000000000000000 P 6a385942aca04c84b16cd85c5407bae0 [term 1 FOLLOWER]: Refusing update from remote peer 16434748810647be9737fc5cf81d3164: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
02:06:21.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:21.559204 8410 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 16434748810647be9737fc5cf81d3164 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6a385942aca04c84b16cd85c5407bae0" member_type: VOTER last_known_addr { host: "127.7.234.126" port: 34333 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
02:06:21.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:21.624465 8158 tablet.cc:2378] T 00000000000000000000000000000000 P 6a385942aca04c84b16cd85c5407bae0: Can't schedule compaction. Clean time has not been advanced past its initial value.
02:06:21.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:21.700551 8423 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6a385942aca04c84b16cd85c5407bae0 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 16434748810647be9737fc5cf81d3164. Latest consensus state: current_term: 1 leader_uuid: "16434748810647be9737fc5cf81d3164" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a385942aca04c84b16cd85c5407bae0" member_type: VOTER last_known_addr { host: "127.7.234.126" port: 34333 } } peers { permanent_uuid: "58a5c5945c8a4f35b49aa5dcbed1c006" member_type: VOTER last_known_addr { host: "127.7.234.125" port: 33623 } } peers { permanent_uuid: "16434748810647be9737fc5cf81d3164" member_type: VOTER last_known_addr { host: "127.7.234.124" port: 38601 } } }
02:06:21.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:21.701006 8433 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6a385942aca04c84b16cd85c5407bae0 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "16434748810647be9737fc5cf81d3164" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a385942aca04c84b16cd85c5407bae0" member_type: VOTER last_known_addr { host: "127.7.234.126" port: 34333 } } peers { permanent_uuid: "58a5c5945c8a4f35b49aa5dcbed1c006" member_type: VOTER last_known_addr { host: "127.7.234.125" port: 33623 } } peers { permanent_uuid: "16434748810647be9737fc5cf81d3164" member_type: VOTER last_known_addr { host: "127.7.234.124" port: 38601 } } }
02:06:21.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:21.701727 8433 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6a385942aca04c84b16cd85c5407bae0 [sys.catalog]: This master's current role is: FOLLOWER
02:06:21.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:21.701565 8423 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6a385942aca04c84b16cd85c5407bae0 [sys.catalog]: This master's current role is: FOLLOWER
02:06:21.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:21.707991 8439 catalog_manager.cc:797] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
02:06:21.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:21.850584 8421 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:06:21.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:21.851227 8421 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:06:21.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:21.852254 8421 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:06:21.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:21.877846 8408 catalog_manager.cc:1261] Loaded cluster ID: c5aad08e8e8643a6baa349b9d79cbd4e
02:06:21.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:21.878645 8408 catalog_manager.cc:1554] T 00000000000000000000000000000000 P 58a5c5945c8a4f35b49aa5dcbed1c006: loading cluster ID for follower catalog manager: success
02:06:21.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:21.886775 8408 catalog_manager.cc:1576] T 00000000000000000000000000000000 P 58a5c5945c8a4f35b49aa5dcbed1c006: acquiring CA information for follower catalog manager: success
02:06:21.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:21.894387 8408 catalog_manager.cc:1604] T 00000000000000000000000000000000 P 58a5c5945c8a4f35b49aa5dcbed1c006: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
02:06:21.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:21.913379 8421 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:06:21.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:21.915000 8421 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.7.234.65
02:06:21.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:21.964054 8421 tablet_server_runner.cc:78] Tablet server non-default flags:
02:06:21.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:06:21.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.234.84:33745
02:06:21.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:06:21.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:06:21.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/ts-0/data
02:06:21.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/ts-0/wal
02:06:21.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/krb5kdc/kudu_127.7.234.65.keytab
02:06:21.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.7.234.65
02:06:21.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:06:21.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:06:21.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.234.65:0
02:06:21.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:06:21.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:06:21.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
02:06:21.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
02:06:21.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:06:21.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:06:21.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/ts-0/data/info.pb
02:06:21.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.234.65
02:06:21.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:06:21.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.7.234.126:34333,127.7.234.125:33623,127.7.234.124:38601
02:06:21.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:06:21.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:06:21.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:06:21.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:06:21.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:06:21.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.7.234.65
02:06:21.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/ts-0/logs
02:06:21.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:06:21.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:06:21.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:06:21.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
02:06:21.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
02:06:21.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision f0776864a65c19ebdd909053f3e624e8bcc25cee
02:06:21.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:06:21.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 28 Jun 2025 01:43:19 UTC on 5fd53c4cbb9d
02:06:21.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 6805
02:06:21.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:06:21.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:21.965691 8421 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:06:21.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:21.967108 8421 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:06:21.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:21.990326 8445 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:22.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:22.138351 8452 sys_catalog.cc:455] T 00000000000000000000000000000000 P 16434748810647be9737fc5cf81d3164 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "16434748810647be9737fc5cf81d3164" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a385942aca04c84b16cd85c5407bae0" member_type: VOTER last_known_addr { host: "127.7.234.126" port: 34333 } } peers { permanent_uuid: "58a5c5945c8a4f35b49aa5dcbed1c006" member_type: VOTER last_known_addr { host: "127.7.234.125" port: 33623 } } peers { permanent_uuid: "16434748810647be9737fc5cf81d3164" member_type: VOTER last_known_addr { host: "127.7.234.124" port: 38601 } } }
02:06:22.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:22.139076 8452 sys_catalog.cc:458] T 00000000000000000000000000000000 P 16434748810647be9737fc5cf81d3164 [sys.catalog]: This master's current role is: LEADER
02:06:22.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:22.725289 8439 catalog_manager.cc:1261] Loaded cluster ID: c5aad08e8e8643a6baa349b9d79cbd4e
02:06:22.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:22.725580 8439 catalog_manager.cc:1554] T 00000000000000000000000000000000 P 6a385942aca04c84b16cd85c5407bae0: loading cluster ID for follower catalog manager: success
02:06:22.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:22.770120 8439 catalog_manager.cc:1576] T 00000000000000000000000000000000 P 6a385942aca04c84b16cd85c5407bae0: acquiring CA information for follower catalog manager: success
02:06:22.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:22.781131 8439 catalog_manager.cc:1604] T 00000000000000000000000000000000 P 6a385942aca04c84b16cd85c5407bae0: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
02:06:25.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:21.994479 8446 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:25.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:25.425271 8421 thread.cc:640] OpenStack (cloud detector) Time spent creating pthread: real 3.433s user 0.991s sys 1.758s
02:06:25.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:23.383785 8444 debug-util.cc:398] Leaking SignalData structure 0x7b0800018d80 after lost signal to thread 8421
02:06:25.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:25.426226 8421 thread.cc:606] OpenStack (cloud detector) Time spent starting thread: real 3.434s user 0.991s sys 1.758s
02:06:25.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:25.428437 8448 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:25.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:25.432864 8447 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 3439 milliseconds
02:06:25.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:25.432868 8421 server_base.cc:1043] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:06:25.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:25.434151 8421 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:06:25.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:25.436518 8421 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:06:25.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:25.437861 8421 hybrid_clock.cc:648] HybridClock initialized: now 1751076385437816 us; error 66 us; skew 500 ppm
02:06:25.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:25.438666 8421 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:06:25.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 28 02:06:25 dist-test-slave-lq1z krb5kdc[8119](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1751076385, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.234.65@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:06:25.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:25.446969 8421 init.cc:377] Logged in from keytab as kudu/127.7.234.65@KRBTEST.COM (short username kudu)
02:06:25.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:25.453503 8421 webserver.cc:469] Webserver started at http://127.7.234.65:44761/ using document root <none> and password file <none>
02:06:25.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:25.454690 8421 fs_manager.cc:362] Metadata directory not provided
02:06:25.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:25.454949 8421 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:06:25.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:25.455456 8421 server_base.cc:896] This appears to be a new deployment of Kudu; creating new FS layout
02:06:25.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:25.461802 8421 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/ts-0/data/instance:
02:06:25.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8e3353a7494549b49fde87660476a705"
02:06:25.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-28 02:06:25 on dist-test-slave-lq1z"
02:06:25.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:25.463279 8421 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/ts-0/wal/instance:
02:06:25.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8e3353a7494549b49fde87660476a705"
02:06:25.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-28 02:06:25 on dist-test-slave-lq1z"
02:06:25.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:25.473740 8421 fs_manager.cc:696] Time spent creating directory manager: real 0.010s user 0.006s sys 0.001s
02:06:25.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:25.483350 8468 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:06:25.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:25.484728 8421 fs_manager.cc:730] Time spent opening block manager: real 0.007s user 0.006s sys 0.001s
02:06:25.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:25.485097 8421 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/ts-0/data,/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/ts-0/wal
02:06:25.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8e3353a7494549b49fde87660476a705"
02:06:25.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-28 02:06:25 on dist-test-slave-lq1z"
02:06:25.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:25.485534 8421 fs_report.cc:389] FS layout report
02:06:25.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:06:25.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/ts-0/wal
02:06:25.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/ts-0/wal
02:06:25.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/ts-0/data/data
02:06:25.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:06:25.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:06:25.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:06:25.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:06:25.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:06:25.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:06:25.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:06:25.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:06:25.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:06:25.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:06:25.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:06:25.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:25.544245 8421 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:06:25.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:25.550232 8421 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:06:25.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:25.550637 8421 kserver.cc:163] Server-wide thread pool size limit: 3276
02:06:25.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:25.553242 8421 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
02:06:25.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:25.557946 8421 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
02:06:25.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:25.558157 8421 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
02:06:25.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:25.558437 8421 ts_tablet_manager.cc:610] Registered 0 tablets
02:06:25.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:25.558620 8421 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
02:06:25.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:25.773998 8580 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.7.234.65:38803 every 8 connection(s)
02:06:25.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:25.773917 8421 rpc_server.cc:307] RPC server started. Bound to: 127.7.234.65:38803
02:06:25.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:25.779062 8421 server_base.cc:1180] Dumped server information to /tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/ts-0/data/info.pb
02:06:25.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:25.784727 8105 external_mini_cluster.cc:1413] Started /tmp/dist-test-task8JcgIg/build/tsan/bin/kudu as pid 8421
02:06:25.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:25.785351 8105 external_mini_cluster.cc:1427] Reading /tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/ts-0/wal/instance
02:06:25.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.7.234.66@KRBTEST.COM; defaulting to no policy
02:06:25.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 28 02:06:25 dist-test-slave-lq1z krb5kdc[8119](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1751076385, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.234.65@KRBTEST.COM for kudu/127.7.234.124@KRBTEST.COM
02:06:25.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:06:25.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.7.234.66@KRBTEST.COM" created.
02:06:25.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 28 02:06:25 dist-test-slave-lq1z krb5kdc[8119](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1751076385, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.234.65@KRBTEST.COM for kudu/127.7.234.126@KRBTEST.COM
02:06:25.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 28 02:06:25 dist-test-slave-lq1z krb5kdc[8119](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1751076385, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.234.65@KRBTEST.COM for kudu/127.7.234.125@KRBTEST.COM
02:06:25.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:25.957770 8582 heartbeater.cc:344] Connected to a master server at 127.7.234.125:33623
02:06:25.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:25.958300 8582 heartbeater.cc:461] Registering TS with master...
02:06:25.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:25.958923 8585 heartbeater.cc:344] Connected to a master server at 127.7.234.126:34333
02:06:25.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:25.959257 8585 heartbeater.cc:461] Registering TS with master...
02:06:25.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:25.959652 8582 heartbeater.cc:507] Master 127.7.234.125:33623 requested a full tablet report, sending...
02:06:25.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:25.961047 8585 heartbeater.cc:507] Master 127.7.234.126:34333 requested a full tablet report, sending...
02:06:25.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:25.961560 8581 heartbeater.cc:344] Connected to a master server at 127.7.234.124:38601
02:06:25.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:25.961903 8581 heartbeater.cc:461] Registering TS with master...
02:06:25.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:25.962787 8581 heartbeater.cc:507] Master 127.7.234.124:38601 requested a full tablet report, sending...
02:06:25.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:25.965406 8169 ts_manager.cc:194] Registered new tserver with Master: 8e3353a7494549b49fde87660476a705 (127.7.234.65:38803)
02:06:25.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:25.966387 8249 ts_manager.cc:194] Registered new tserver with Master: 8e3353a7494549b49fde87660476a705 (127.7.234.65:38803)
02:06:25.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:25.968932 8335 ts_manager.cc:194] Registered new tserver with Master: 8e3353a7494549b49fde87660476a705 (127.7.234.65:38803)
02:06:25.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:25.971558 8335 master_service.cc:496] Signed X509 certificate for tserver {username='kudu', principal='kudu/127.7.234.65@KRBTEST.COM'} at 127.7.234.65:52597
02:06:25.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:06:25.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.7.234.66 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/krb5kdc/kudu_127.7.234.66.keytab.
02:06:25.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.7.234.66 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/krb5kdc/kudu_127.7.234.66.keytab.
02:06:25.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:25.973713 8105 mini_kdc.cc:262] Time spent creating service keytab for kudu/127.7.234.66: real 0.165s user 0.003s sys 0.010s
02:06:25.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:25.975840 8105 external_mini_cluster.cc:1351] Running /tmp/dist-test-task8JcgIg/build/tsan/bin/kudu
02:06:25.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task8JcgIg/build/tsan/bin/kudu
02:06:25.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/ts-1/wal
02:06:25.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/ts-1/data
02:06:25.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:06:25.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:06:25.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:06:25.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:06:25.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:06:25.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:06:25.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/ts-1/logs
02:06:25.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/ts-1/data/info.pb
02:06:25.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:06:25.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:06:25.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:06:25.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:06:25.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:06:25.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:06:25.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:06:25.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:06:25.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
02:06:25.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:06:25.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.234.66:0
02:06:25.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.7.234.66
02:06:25.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.234.66
02:06:25.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:06:25.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.7.234.126:34333,127.7.234.125:33623,127.7.234.124:38601
02:06:25.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/krb5kdc/kudu_127.7.234.66.keytab
02:06:25.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.7.234.66
02:06:25.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:06:25.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
02:06:25.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
02:06:25.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.234.84:33745
02:06:25.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:06:25.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:06:25.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {KRB5CCNAME=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
02:06:26.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 28 02:06:26 dist-test-slave-lq1z krb5kdc[8119](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1751076386, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.234.126@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:06:26.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:26.279814 8148 init.cc:295] Successfully reacquired a new kerberos TGT
02:06:26.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:26.330389 8593 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:06:26.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:26.330890 8593 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:06:26.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:26.331480 8593 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:06:26.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:26.365645 8593 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:06:26.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:26.366603 8593 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.7.234.66
02:06:26.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:26.403699 8593 tablet_server_runner.cc:78] Tablet server non-default flags:
02:06:26.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:06:26.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.234.84:33745
02:06:26.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:06:26.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:06:26.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/ts-1/data
02:06:26.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/ts-1/wal
02:06:26.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/krb5kdc/kudu_127.7.234.66.keytab
02:06:26.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.7.234.66
02:06:26.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:06:26.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:06:26.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.234.66:0
02:06:26.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:06:26.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:06:26.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
02:06:26.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
02:06:26.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:06:26.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:06:26.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/ts-1/data/info.pb
02:06:26.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.234.66
02:06:26.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:06:26.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.7.234.126:34333,127.7.234.125:33623,127.7.234.124:38601
02:06:26.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:06:26.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:06:26.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:06:26.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:06:26.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:06:26.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.7.234.66
02:06:26.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/ts-1/logs
02:06:26.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:06:26.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:06:26.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:06:26.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
02:06:26.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
02:06:26.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision f0776864a65c19ebdd909053f3e624e8bcc25cee
02:06:26.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:06:26.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 28 Jun 2025 01:43:19 UTC on 5fd53c4cbb9d
02:06:26.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 6805
02:06:26.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:06:26.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:26.405048 8593 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:06:26.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:26.406102 8593 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:06:26.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:26.424368 8600 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:26.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:26.975071 8581 heartbeater.cc:499] Master 127.7.234.124:38601 was elected leader, sending a full tablet report...
02:06:28.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 28 02:06:28 dist-test-slave-lq1z krb5kdc[8119](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1751076388, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.234.125@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:06:28.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:28.168598 8227 init.cc:295] Successfully reacquired a new kerberos TGT
02:06:28.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:28.193702 8593 thread.cc:640] OpenStack (cloud detector) Time spent creating pthread: real 1.774s user 0.000s sys 0.002s
02:06:28.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:28.194190 8593 thread.cc:606] OpenStack (cloud detector) Time spent starting thread: real 1.774s user 0.000s sys 0.002s
02:06:28.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:28.194371 8601 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1772 milliseconds
02:06:28.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:27.823324 8598 debug-util.cc:398] Leaking SignalData structure 0x7b0800018d80 after lost signal to thread 8593
02:06:28.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:26.424505 8599 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:28.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:28.196058 8593 server_base.cc:1043] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:06:28.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:28.196095 8602 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:28.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:28.199908 8593 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:06:28.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:28.202018 8593 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:06:28.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:28.203344 8593 hybrid_clock.cc:648] HybridClock initialized: now 1751076388203301 us; error 49 us; skew 500 ppm
02:06:28.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:28.204195 8593 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:06:28.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 28 02:06:28 dist-test-slave-lq1z krb5kdc[8119](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1751076388, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.234.66@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:06:28.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:28.210067 8593 init.cc:377] Logged in from keytab as kudu/127.7.234.66@KRBTEST.COM (short username kudu)
02:06:28.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:28.215159 8593 webserver.cc:469] Webserver started at http://127.7.234.66:46435/ using document root <none> and password file <none>
02:06:28.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:28.216071 8593 fs_manager.cc:362] Metadata directory not provided
02:06:28.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:28.216303 8593 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:06:28.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:28.216789 8593 server_base.cc:896] This appears to be a new deployment of Kudu; creating new FS layout
02:06:28.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:28.221385 8593 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/ts-1/data/instance:
02:06:28.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "43991169d0e84efcb4f4b7bca98ff922"
02:06:28.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-28 02:06:28 on dist-test-slave-lq1z"
02:06:28.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:28.222488 8593 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/ts-1/wal/instance:
02:06:28.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "43991169d0e84efcb4f4b7bca98ff922"
02:06:28.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-28 02:06:28 on dist-test-slave-lq1z"
02:06:28.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:28.229795 8593 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.005s sys 0.003s
02:06:28.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:28.234977 8615 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:06:28.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:28.236063 8593 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.002s sys 0.000s
02:06:28.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:28.236354 8593 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/ts-1/data,/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/ts-1/wal
02:06:28.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "43991169d0e84efcb4f4b7bca98ff922"
02:06:28.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-28 02:06:28 on dist-test-slave-lq1z"
02:06:28.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:28.236657 8593 fs_report.cc:389] FS layout report
02:06:28.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:06:28.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/ts-1/wal
02:06:28.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/ts-1/wal
02:06:28.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/ts-1/data/data
02:06:28.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:06:28.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:06:28.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:06:28.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:06:28.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:06:28.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:06:28.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:06:28.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:06:28.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:06:28.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:06:28.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:06:28.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:28.284001 8593 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:06:28.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:28.290330 8593 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:06:28.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:28.290827 8593 kserver.cc:163] Server-wide thread pool size limit: 3276
02:06:28.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:28.293452 8593 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
02:06:28.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:28.297993 8593 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
02:06:28.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:28.298297 8593 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
02:06:28.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:28.298628 8593 ts_tablet_manager.cc:610] Registered 0 tablets
02:06:28.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:28.298818 8593 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
02:06:28.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:28.472231 8593 rpc_server.cc:307] RPC server started. Bound to: 127.7.234.66:43479
02:06:28.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:28.472352 8729 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.7.234.66:43479 every 8 connection(s)
02:06:28.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:28.474777 8593 server_base.cc:1180] Dumped server information to /tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/ts-1/data/info.pb
02:06:28.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:28.480402 8105 external_mini_cluster.cc:1413] Started /tmp/dist-test-task8JcgIg/build/tsan/bin/kudu as pid 8593
02:06:28.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:28.481024 8105 external_mini_cluster.cc:1427] Reading /tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/ts-1/wal/instance
02:06:28.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 28 02:06:28 dist-test-slave-lq1z krb5kdc[8119](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1751076388, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.234.66@KRBTEST.COM for kudu/127.7.234.125@KRBTEST.COM
02:06:28.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 28 02:06:28 dist-test-slave-lq1z krb5kdc[8119](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1751076388, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.234.66@KRBTEST.COM for kudu/127.7.234.124@KRBTEST.COM
02:06:28.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 28 02:06:28 dist-test-slave-lq1z krb5kdc[8119](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1751076388, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.234.66@KRBTEST.COM for kudu/127.7.234.126@KRBTEST.COM
02:06:28.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.7.234.67@KRBTEST.COM; defaulting to no policy
02:06:28.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:28.546988 8731 heartbeater.cc:344] Connected to a master server at 127.7.234.125:33623
02:06:28.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:28.547187 8730 heartbeater.cc:344] Connected to a master server at 127.7.234.124:38601
02:06:28.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:28.547539 8731 heartbeater.cc:461] Registering TS with master...
02:06:28.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:28.547741 8730 heartbeater.cc:461] Registering TS with master...
02:06:28.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:28.549052 8731 heartbeater.cc:507] Master 127.7.234.125:33623 requested a full tablet report, sending...
02:06:28.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:28.549052 8730 heartbeater.cc:507] Master 127.7.234.124:38601 requested a full tablet report, sending...
02:06:28.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:28.552385 8249 ts_manager.cc:194] Registered new tserver with Master: 43991169d0e84efcb4f4b7bca98ff922 (127.7.234.66:43479)
02:06:28.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:28.552403 8335 ts_manager.cc:194] Registered new tserver with Master: 43991169d0e84efcb4f4b7bca98ff922 (127.7.234.66:43479)
02:06:28.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:28.553900 8335 master_service.cc:496] Signed X509 certificate for tserver {username='kudu', principal='kudu/127.7.234.66@KRBTEST.COM'} at 127.7.234.66:58123
02:06:28.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:28.554983 8732 heartbeater.cc:344] Connected to a master server at 127.7.234.126:34333
02:06:28.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:28.555285 8732 heartbeater.cc:461] Registering TS with master...
02:06:28.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:28.556171 8732 heartbeater.cc:507] Master 127.7.234.126:34333 requested a full tablet report, sending...
02:06:28.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:28.559011 8169 ts_manager.cc:194] Registered new tserver with Master: 43991169d0e84efcb4f4b7bca98ff922 (127.7.234.66:43479)
02:06:28.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:06:28.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.7.234.67@KRBTEST.COM" created.
02:06:28.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:06:28.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.7.234.67 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/krb5kdc/kudu_127.7.234.67.keytab.
02:06:28.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.7.234.67 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/krb5kdc/kudu_127.7.234.67.keytab.
02:06:28.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:28.585971 8105 external_mini_cluster.cc:1351] Running /tmp/dist-test-task8JcgIg/build/tsan/bin/kudu
02:06:28.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task8JcgIg/build/tsan/bin/kudu
02:06:28.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/ts-2/wal
02:06:28.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/ts-2/data
02:06:28.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:06:28.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:06:28.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:06:28.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:06:28.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:06:28.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:06:28.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/ts-2/logs
02:06:28.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/ts-2/data/info.pb
02:06:28.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:06:28.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:06:28.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:06:28.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:06:28.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:06:28.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:06:28.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:06:28.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:06:28.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
02:06:28.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:06:28.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.234.67:0
02:06:28.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.7.234.67
02:06:28.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.234.67
02:06:28.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:06:28.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.7.234.126:34333,127.7.234.125:33623,127.7.234.124:38601
02:06:28.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/krb5kdc/kudu_127.7.234.67.keytab
02:06:28.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.7.234.67
02:06:28.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:06:28.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
02:06:28.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
02:06:28.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.234.84:33745
02:06:28.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:06:28.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:06:28.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {KRB5CCNAME=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
02:06:28.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:28.911504 8742 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:06:28.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:28.912073 8742 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:06:28.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:28.912657 8742 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:06:28.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:28.946544 8742 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:06:28.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:28.947455 8742 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.7.234.67
02:06:28.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:28.984685 8742 tablet_server_runner.cc:78] Tablet server non-default flags:
02:06:28.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:06:28.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.234.84:33745
02:06:28.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:06:28.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:06:28.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/ts-2/data
02:06:28.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/ts-2/wal
02:06:28.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/krb5kdc/kudu_127.7.234.67.keytab
02:06:28.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.7.234.67
02:06:28.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:06:28.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:06:28.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.234.67:0
02:06:28.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:06:28.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:06:28.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
02:06:28.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
02:06:28.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:06:28.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:06:28.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/ts-2/data/info.pb
02:06:28.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.234.67
02:06:28.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:06:28.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.7.234.126:34333,127.7.234.125:33623,127.7.234.124:38601
02:06:28.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:06:28.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:06:28.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:06:28.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:06:28.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:06:28.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.7.234.67
02:06:28.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/ts-2/logs
02:06:28.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:06:28.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:06:28.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:06:28.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
02:06:28.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
02:06:28.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision f0776864a65c19ebdd909053f3e624e8bcc25cee
02:06:28.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:06:28.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 28 Jun 2025 01:43:19 UTC on 5fd53c4cbb9d
02:06:28.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 6805
02:06:28.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:06:28.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:28.985924 8742 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:06:28.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:28.986984 8742 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:06:29.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:29.006107 8751 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:29.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:29.557531 8730 heartbeater.cc:499] Master 127.7.234.124:38601 was elected leader, sending a full tablet report...
02:06:30.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 28 02:06:30 dist-test-slave-lq1z krb5kdc[8119](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1751076390, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.234.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:06:30.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:30.102373 8314 init.cc:295] Successfully reacquired a new kerberos TGT
02:06:30.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:29.006801 8753 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:30.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:29.006218 8750 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:30.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:30.220958 8752 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
02:06:30.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:30.221519 8742 server_base.cc:1043] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:06:30.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:30.225333 8742 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:06:30.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:30.228389 8742 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:06:30.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:30.229755 8742 hybrid_clock.cc:648] HybridClock initialized: now 1751076390229708 us; error 53 us; skew 500 ppm
02:06:30.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:30.230566 8742 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:06:30.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 28 02:06:30 dist-test-slave-lq1z krb5kdc[8119](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1751076390, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.234.67@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:06:30.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:30.242077 8742 init.cc:377] Logged in from keytab as kudu/127.7.234.67@KRBTEST.COM (short username kudu)
02:06:30.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:30.247890 8742 webserver.cc:469] Webserver started at http://127.7.234.67:39901/ using document root <none> and password file <none>
02:06:30.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:30.248804 8742 fs_manager.cc:362] Metadata directory not provided
02:06:30.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:30.248996 8742 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:06:30.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:30.249526 8742 server_base.cc:896] This appears to be a new deployment of Kudu; creating new FS layout
02:06:30.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:30.253962 8742 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/ts-2/data/instance:
02:06:30.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fb8d9e2f1452456e88e19234cb1b835f"
02:06:30.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-28 02:06:30 on dist-test-slave-lq1z"
02:06:30.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:30.255134 8742 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/ts-2/wal/instance:
02:06:30.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fb8d9e2f1452456e88e19234cb1b835f"
02:06:30.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-28 02:06:30 on dist-test-slave-lq1z"
02:06:30.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:30.262578 8742 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.005s sys 0.000s
02:06:30.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:30.268488 8762 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:06:30.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:30.269682 8742 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.004s sys 0.000s
02:06:30.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:30.270052 8742 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/ts-2/data,/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/ts-2/wal
02:06:30.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fb8d9e2f1452456e88e19234cb1b835f"
02:06:30.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-28 02:06:30 on dist-test-slave-lq1z"
02:06:30.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:30.270426 8742 fs_report.cc:389] FS layout report
02:06:30.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:06:30.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/ts-2/wal
02:06:30.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/ts-2/wal
02:06:30.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/ts-2/data/data
02:06:30.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:06:30.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:06:30.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:06:30.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:06:30.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:06:30.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:06:30.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:06:30.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:06:30.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:06:30.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:06:30.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:06:30.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:30.323674 8742 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:06:30.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:30.329967 8742 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:06:30.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:30.330353 8742 kserver.cc:163] Server-wide thread pool size limit: 3276
02:06:30.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:30.333438 8742 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
02:06:30.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:30.338281 8742 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
02:06:30.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:30.338506 8742 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
02:06:30.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:30.338819 8742 ts_tablet_manager.cc:610] Registered 0 tablets
02:06:30.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:30.338985 8742 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
02:06:30.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:30.659550 8742 rpc_server.cc:307] RPC server started. Bound to: 127.7.234.67:45767
02:06:30.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:30.659618 8875 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.7.234.67:45767 every 8 connection(s)
02:06:30.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:30.662477 8742 server_base.cc:1180] Dumped server information to /tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/ts-2/data/info.pb
02:06:30.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:30.668376 8105 external_mini_cluster.cc:1413] Started /tmp/dist-test-task8JcgIg/build/tsan/bin/kudu as pid 8742
02:06:30.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:30.668869 8105 external_mini_cluster.cc:1427] Reading /tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/ts-2/wal/instance
02:06:30.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 28 02:06:30 dist-test-slave-lq1z krb5kdc[8119](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1751076390, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.234.67@KRBTEST.COM for kudu/127.7.234.124@KRBTEST.COM
02:06:30.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 28 02:06:30 dist-test-slave-lq1z krb5kdc[8119](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1751076390, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.234.67@KRBTEST.COM for kudu/127.7.234.125@KRBTEST.COM
02:06:30.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 28 02:06:30 dist-test-slave-lq1z krb5kdc[8119](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1751076390, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.234.67@KRBTEST.COM for kudu/127.7.234.126@KRBTEST.COM
02:06:30.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:30.734212 8876 heartbeater.cc:344] Connected to a master server at 127.7.234.124:38601
02:06:30.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:30.734741 8876 heartbeater.cc:461] Registering TS with master...
02:06:30.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:30.736560 8876 heartbeater.cc:507] Master 127.7.234.124:38601 requested a full tablet report, sending...
02:06:30.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:30.738420 8877 heartbeater.cc:344] Connected to a master server at 127.7.234.125:33623
02:06:30.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:30.738739 8877 heartbeater.cc:461] Registering TS with master...
02:06:30.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:30.739651 8877 heartbeater.cc:507] Master 127.7.234.125:33623 requested a full tablet report, sending...
02:06:30.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:30.742494 8335 ts_manager.cc:194] Registered new tserver with Master: fb8d9e2f1452456e88e19234cb1b835f (127.7.234.67:45767)
02:06:30.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:30.743794 8248 ts_manager.cc:194] Registered new tserver with Master: fb8d9e2f1452456e88e19234cb1b835f (127.7.234.67:45767)
02:06:30.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:30.744129 8880 heartbeater.cc:344] Connected to a master server at 127.7.234.126:34333
02:06:30.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:30.744557 8880 heartbeater.cc:461] Registering TS with master...
02:06:30.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:30.744500 8335 master_service.cc:496] Signed X509 certificate for tserver {username='kudu', principal='kudu/127.7.234.67@KRBTEST.COM'} at 127.7.234.67:33757
02:06:30.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:30.745571 8880 heartbeater.cc:507] Master 127.7.234.126:34333 requested a full tablet report, sending...
02:06:30.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:30.747745 8169 ts_manager.cc:194] Registered new tserver with Master: fb8d9e2f1452456e88e19234cb1b835f (127.7.234.67:45767)
02:06:30.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:30.751734 8105 external_mini_cluster.cc:934] 3 TS(s) registered with all masters
02:06:30.764 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:06:30.765 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
02:06:30.778 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
02:06:30.779 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
02:06:30.782 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.7.234.126"
port: 34333
}
bound_http_address {
host: "127.7.234.126"
port: 37231
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.7.234.125"
port: 33623
}
bound_http_address {
host: "127.7.234.125"
port: 38081
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.7.234.124"
port: 38601
}
bound_http_address {
host: "127.7.234.124"
port: 40485
}
}
}
02:06:30.782 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
02:06:30.784 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.7.234.65"
port: 38803
}
bound_http_address {
host: "127.7.234.65"
port: 44761
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.7.234.66"
port: 43479
}
bound_http_address {
host: "127.7.234.66"
port: 46435
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.7.234.67"
port: 45767
}
bound_http_address {
host: "127.7.234.67"
port: 39901
}
}
}
02:06:30.784 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
02:06:30.785 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
02:06:30.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 28 02:06:30 dist-test-slave-lq1z krb5kdc[8119](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1751076390, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:06:30.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
02:06:30.824 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:06:30.826 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/krb5kdc/krb5cc
02:06:30.831 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
02:06:30.832 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/krb5kdc/krb5cc
02:06:30.833 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
02:06:30.837 [DEBUG - Time-limited test] (SecurityContext.java:158) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
02:06:30.839 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
02:06:30.839 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
02:06:30.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 28 02:06:30 dist-test-slave-lq1z krb5kdc[8119](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1751076390, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:06:30.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
02:06:30.870 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:06:30.870 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/krb5kdc/krb5cc
02:06:30.871 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
02:06:30.872 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.7.234.126:34333
02:06:30.873 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
02:06:30.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:30.874087 8105 external_mini_cluster.cc:1620] Killing /tmp/dist-test-task8JcgIg/build/tsan/bin/kudu with pid 8134
02:06:30.904 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:06:30.904 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.7.234.125:33623
02:06:30.904 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
02:06:30.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:30.905054 8105 external_mini_cluster.cc:1620] Killing /tmp/dist-test-task8JcgIg/build/tsan/bin/kudu with pid 8210
02:06:30.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:30.936298 8319 connection.cc:537] client connection to 127.7.234.125:33623 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
02:06:30.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:30.936798 8319 proxy.cc:239] Call had error, refreshing address and retrying: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
02:06:30.938 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:06:30.938 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.7.234.124:38601
02:06:30.938 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
02:06:30.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:30.939139 8105 external_mini_cluster.cc:1620] Killing /tmp/dist-test-task8JcgIg/build/tsan/bin/kudu with pid 8291
02:06:30.976 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:06:30.976 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.7.234.126:34333
02:06:30.982 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
02:06:30.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:30.984520 8105 external_mini_cluster.cc:1351] Running /tmp/dist-test-task8JcgIg/build/tsan/bin/kudu
02:06:30.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task8JcgIg/build/tsan/bin/kudu
02:06:30.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/master-0/wal
02:06:30.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/master-0/data
02:06:30.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:06:30.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:06:30.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:06:30.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:06:30.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:06:30.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:06:30.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/master-0/logs
02:06:30.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/master-0/data/info.pb
02:06:30.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:06:30.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:06:30.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:06:30.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:06:30.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:06:30.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:06:30.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:06:30.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:06:30.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:06:30.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:06:30.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:06:30.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:06:30.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.234.126:34333
02:06:30.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.234.126
02:06:30.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37231
02:06:30.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/krb5kdc/kudu_127.7.234.126.keytab
02:06:30.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.7.234.126
02:06:30.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:06:30.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
02:06:30.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
02:06:30.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.234.84:33745
02:06:30.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:06:30.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:06:30.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:06:30.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:06:30.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.234.126:34333,127.7.234.125:33623,127.7.234.124:38601
02:06:30.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:06:30.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35625/jwks.json
02:06:30.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
02:06:31.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:31.005111 8581 heartbeater.cc:646] Failed to heartbeat to 127.7.234.124:38601 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.7.234.124:38601: connect: Connection refused (error 111)
02:06:31.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:31.305228 8891 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:06:31.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:31.305897 8891 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:06:31.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:31.306617 8891 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:06:31.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:31.339175 8891 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
02:06:31.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:31.339490 8891 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
02:06:31.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:31.339915 8891 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:06:31.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:31.340256 8891 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
02:06:31.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:31.340566 8891 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
02:06:31.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:31.340868 8891 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
02:06:31.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:31.341135 8891 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:35625/jwks.json
02:06:31.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:31.380657 8891 master_runner.cc:386] Master server non-default flags:
02:06:31.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:06:31.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.234.84:33745
02:06:31.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:06:31.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:06:31.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/master-0/data
02:06:31.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/master-0/wal
02:06:31.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
02:06:31.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:06:31.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.234.126:34333,127.7.234.125:33623,127.7.234.124:38601
02:06:31.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/krb5kdc/kudu_127.7.234.126.keytab
02:06:31.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.7.234.126
02:06:31.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:06:31.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:06:31.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:06:31.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.234.126:34333
02:06:31.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:06:31.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:06:31.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:06:31.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35625/jwks.json
02:06:31.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:06:31.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
02:06:31.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
02:06:31.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:06:31.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:06:31.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/master-0/data/info.pb
02:06:31.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.234.126
02:06:31.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37231
02:06:31.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:06:31.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:06:31.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:06:31.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:06:31.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:06:31.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/master-0/logs
02:06:31.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:06:31.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:06:31.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:06:31.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:06:31.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
02:06:31.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision f0776864a65c19ebdd909053f3e624e8bcc25cee
02:06:31.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:06:31.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 28 Jun 2025 01:43:19 UTC on 5fd53c4cbb9d
02:06:31.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 6805
02:06:31.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:06:31.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:31.382359 8891 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:06:31.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:31.383706 8891 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:06:31.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:31.398634 8900 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:31.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:31.565297 8731 heartbeater.cc:646] Failed to heartbeat to 127.7.234.125:33623 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.7.234.125:33623: connect: Connection refused (error 111)
02:06:31.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:31.749394 8876 heartbeater.cc:499] Master 127.7.234.124:38601 was elected leader, sending a full tablet report...
02:06:31.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:31.766237 8880 heartbeater.cc:646] Failed to heartbeat to 127.7.234.126:34333 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.7.234.126:34333: connect: Connection refused (error 111)
02:06:32.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:31.398751 8901 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:32.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:31.399518 8903 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:32.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:31.401095 8891 server_base.cc:1048] running on GCE node
02:06:32.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:32.660940 8891 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:06:32.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:32.664331 8891 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:06:32.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:32.665787 8891 hybrid_clock.cc:648] HybridClock initialized: now 1751076392665656 us; error 135 us; skew 500 ppm
02:06:32.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:32.666878 8891 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:06:32.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 28 02:06:32 dist-test-slave-lq1z krb5kdc[8119](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1751076392, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.234.126@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:06:32.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:32.674078 8891 init.cc:377] Logged in from keytab as kudu/127.7.234.126@KRBTEST.COM (short username kudu)
02:06:32.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:32.680660 8891 webserver.cc:469] Webserver started at http://127.7.234.126:37231/ using document root <none> and password file <none>
02:06:32.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:32.681778 8891 fs_manager.cc:362] Metadata directory not provided
02:06:32.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:32.681988 8891 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:06:32.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:32.689950 8891 fs_manager.cc:714] Time spent opening directory manager: real 0.005s user 0.000s sys 0.004s
02:06:32.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:32.694931 8923 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:06:32.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:32.696166 8891 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.004s sys 0.001s
02:06:32.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:32.696619 8891 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/master-0/data,/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/master-0/wal
02:06:32.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6a385942aca04c84b16cd85c5407bae0"
02:06:32.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-28 02:06:16 on dist-test-slave-lq1z"
02:06:32.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:32.699306 8891 fs_report.cc:389] FS layout report
02:06:32.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:06:32.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/master-0/wal
02:06:32.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/master-0/wal
02:06:32.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/master-0/data/data
02:06:32.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:06:32.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:06:32.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:06:32.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:06:32.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:06:32.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:06:32.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:06:32.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:06:32.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:06:32.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:06:32.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:06:32.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:32.760121 8891 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:06:32.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:32.765388 8891 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:06:32.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:32.765805 8891 kserver.cc:163] Server-wide thread pool size limit: 3276
02:06:32.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:32.855078 8891 rpc_server.cc:307] RPC server started. Bound to: 127.7.234.126:34333
02:06:32.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:32.855177 8977 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.7.234.126:34333 every 8 connection(s)
02:06:32.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:32.857971 8891 server_base.cc:1180] Dumped server information to /tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/master-0/data/info.pb
02:06:32.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:32.862275 8105 external_mini_cluster.cc:1413] Started /tmp/dist-test-task8JcgIg/build/tsan/bin/kudu as pid 8891
02:06:32.863 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:06:32.863 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.7.234.125:33623
02:06:32.863 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
02:06:32.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:32.865168 8105 external_mini_cluster.cc:1351] Running /tmp/dist-test-task8JcgIg/build/tsan/bin/kudu
02:06:32.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task8JcgIg/build/tsan/bin/kudu
02:06:32.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/master-1/wal
02:06:32.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/master-1/data
02:06:32.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:06:32.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:06:32.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:06:32.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:06:32.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:06:32.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:06:32.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/master-1/logs
02:06:32.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/master-1/data/info.pb
02:06:32.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:06:32.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:06:32.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:06:32.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:06:32.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:06:32.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:06:32.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:06:32.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:06:32.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:06:32.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:06:32.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:06:32.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:06:32.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.234.125:33623
02:06:32.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.234.125
02:06:32.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38081
02:06:32.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/krb5kdc/kudu_127.7.234.125.keytab
02:06:32.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.7.234.125
02:06:32.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:06:32.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
02:06:32.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
02:06:32.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.234.84:33745
02:06:32.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:06:32.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:06:32.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:06:32.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:06:32.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.234.126:34333,127.7.234.125:33623,127.7.234.124:38601
02:06:32.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:06:32.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35625/jwks.json
02:06:32.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
02:06:32.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:32.871047 8978 sys_catalog.cc:263] Verifying existing consensus state
02:06:32.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:32.876027 8978 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6a385942aca04c84b16cd85c5407bae0: Bootstrap starting.
02:06:32.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:32.913149 8978 log.cc:826] T 00000000000000000000000000000000 P 6a385942aca04c84b16cd85c5407bae0: Log is configured to *not* fsync() on all Append() calls
02:06:32.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:32.928767 8978 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6a385942aca04c84b16cd85c5407bae0: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
02:06:32.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:32.929672 8978 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6a385942aca04c84b16cd85c5407bae0: Bootstrap complete.
02:06:32.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:32.955488 8978 raft_consensus.cc:357] T 00000000000000000000000000000000 P 6a385942aca04c84b16cd85c5407bae0 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a385942aca04c84b16cd85c5407bae0" member_type: VOTER last_known_addr { host: "127.7.234.126" port: 34333 } } peers { permanent_uuid: "58a5c5945c8a4f35b49aa5dcbed1c006" member_type: VOTER last_known_addr { host: "127.7.234.125" port: 33623 } } peers { permanent_uuid: "16434748810647be9737fc5cf81d3164" member_type: VOTER last_known_addr { host: "127.7.234.124" port: 38601 } }
02:06:32.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:32.957733 8978 raft_consensus.cc:738] T 00000000000000000000000000000000 P 6a385942aca04c84b16cd85c5407bae0 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a385942aca04c84b16cd85c5407bae0, State: Initialized, Role: FOLLOWER
02:06:32.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:32.958676 8978 consensus_queue.cc:260] T 00000000000000000000000000000000 P 6a385942aca04c84b16cd85c5407bae0 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a385942aca04c84b16cd85c5407bae0" member_type: VOTER last_known_addr { host: "127.7.234.126" port: 34333 } } peers { permanent_uuid: "58a5c5945c8a4f35b49aa5dcbed1c006" member_type: VOTER last_known_addr { host: "127.7.234.125" port: 33623 } } peers { permanent_uuid: "16434748810647be9737fc5cf81d3164" member_type: VOTER last_known_addr { host: "127.7.234.124" port: 38601 } }
02:06:32.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:32.961663 8982 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6a385942aca04c84b16cd85c5407bae0 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a385942aca04c84b16cd85c5407bae0" member_type: VOTER last_known_addr { host: "127.7.234.126" port: 34333 } } peers { permanent_uuid: "58a5c5945c8a4f35b49aa5dcbed1c006" member_type: VOTER last_known_addr { host: "127.7.234.125" port: 33623 } } peers { permanent_uuid: "16434748810647be9737fc5cf81d3164" member_type: VOTER last_known_addr { host: "127.7.234.124" port: 38601 } } }
02:06:32.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:32.962536 8982 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6a385942aca04c84b16cd85c5407bae0 [sys.catalog]: This master's current role is: FOLLOWER
02:06:32.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:32.963542 8978 sys_catalog.cc:564] T 00000000000000000000000000000000 P 6a385942aca04c84b16cd85c5407bae0 [sys.catalog]: configured and running, proceeding with master startup.
02:06:33.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:33.003486 8993 catalog_manager.cc:1261] Loaded cluster ID: c5aad08e8e8643a6baa349b9d79cbd4e
02:06:33.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:33.003798 8993 catalog_manager.cc:1554] T 00000000000000000000000000000000 P 6a385942aca04c84b16cd85c5407bae0: loading cluster ID for follower catalog manager: success
02:06:33.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:33.013260 8993 catalog_manager.cc:1576] T 00000000000000000000000000000000 P 6a385942aca04c84b16cd85c5407bae0: acquiring CA information for follower catalog manager: success
02:06:33.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 28 02:06:33 dist-test-slave-lq1z krb5kdc[8119](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1751076392, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.234.126@KRBTEST.COM for kudu/127.7.234.126@KRBTEST.COM
02:06:33.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:33.028872 8993 catalog_manager.cc:1604] T 00000000000000000000000000000000 P 6a385942aca04c84b16cd85c5407bae0: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
02:06:33.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:33.048786 8585 heartbeater.cc:344] Connected to a master server at 127.7.234.126:34333
02:06:33.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:33.048098 8891 master_runner.cc:186] Error getting master registration for 127.7.234.125:33623: Network error: Client connection negotiation failed: client connection to 127.7.234.125:33623: connect: Connection refused (error 111),
02:06:33.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:33.075896 8891 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
02:06:33.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:33.078974 8943 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "8e3353a7494549b49fde87660476a705" instance_seqno: 1751076385722956) as {username='kudu', principal='kudu/127.7.234.65@KRBTEST.COM'} at 127.7.234.65:35905; Asking this server to re-register.
02:06:33.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:33.080767 8585 heartbeater.cc:461] Registering TS with master...
02:06:33.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:33.085098 8943 ts_manager.cc:194] Registered new tserver with Master: 8e3353a7494549b49fde87660476a705 (127.7.234.65:38803)
02:06:33.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:33.226049 8980 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:06:33.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:33.226612 8980 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:06:33.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:33.227267 8980 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:06:33.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:33.259320 8980 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
02:06:33.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:33.259565 8980 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
02:06:33.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:33.259829 8980 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:06:33.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:33.260026 8980 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
02:06:33.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:33.260283 8980 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
02:06:33.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:33.260471 8980 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
02:06:33.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:33.260619 8980 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:35625/jwks.json
02:06:33.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:33.300352 8980 master_runner.cc:386] Master server non-default flags:
02:06:33.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:06:33.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.234.84:33745
02:06:33.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:06:33.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:06:33.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/master-1/data
02:06:33.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/master-1/wal
02:06:33.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
02:06:33.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:06:33.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.234.126:34333,127.7.234.125:33623,127.7.234.124:38601
02:06:33.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/krb5kdc/kudu_127.7.234.125.keytab
02:06:33.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.7.234.125
02:06:33.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:06:33.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:06:33.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:06:33.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.234.125:33623
02:06:33.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:06:33.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:06:33.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:06:33.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35625/jwks.json
02:06:33.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:06:33.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
02:06:33.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
02:06:33.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:06:33.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:06:33.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/master-1/data/info.pb
02:06:33.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.234.125
02:06:33.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38081
02:06:33.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:06:33.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:06:33.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:06:33.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:06:33.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:06:33.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/master-1/logs
02:06:33.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:06:33.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:06:33.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:06:33.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:06:33.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
02:06:33.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision f0776864a65c19ebdd909053f3e624e8bcc25cee
02:06:33.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:06:33.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 28 Jun 2025 01:43:19 UTC on 5fd53c4cbb9d
02:06:33.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 6805
02:06:33.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:06:33.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:33.301749 8980 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:06:33.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:33.303171 8980 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:06:33.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:33.320102 9007 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:33.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:33.626590 8732 heartbeater.cc:344] Connected to a master server at 127.7.234.126:34333
02:06:33.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:33.629601 8943 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "43991169d0e84efcb4f4b7bca98ff922" instance_seqno: 1751076388428833) as {username='kudu', principal='kudu/127.7.234.66@KRBTEST.COM'} at 127.7.234.66:43929; Asking this server to re-register.
02:06:33.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:33.630929 8732 heartbeater.cc:461] Registering TS with master...
02:06:33.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:33.638056 8943 ts_manager.cc:194] Registered new tserver with Master: 43991169d0e84efcb4f4b7bca98ff922 (127.7.234.66:43479)
02:06:33.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:33.829197 8880 heartbeater.cc:344] Connected to a master server at 127.7.234.126:34333
02:06:33.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:33.831981 8943 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "fb8d9e2f1452456e88e19234cb1b835f" instance_seqno: 1751076390602995) as {username='kudu', principal='kudu/127.7.234.67@KRBTEST.COM'} at 127.7.234.67:57243; Asking this server to re-register.
02:06:33.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:33.833886 8880 heartbeater.cc:461] Registering TS with master...
02:06:33.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:33.836951 8943 ts_manager.cc:194] Registered new tserver with Master: fb8d9e2f1452456e88e19234cb1b835f (127.7.234.67:45767)
02:06:34.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:34.656455 9023 raft_consensus.cc:491] T 00000000000000000000000000000000 P 6a385942aca04c84b16cd85c5407bae0 [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:06:34.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:34.658052 9023 raft_consensus.cc:513] T 00000000000000000000000000000000 P 6a385942aca04c84b16cd85c5407bae0 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a385942aca04c84b16cd85c5407bae0" member_type: VOTER last_known_addr { host: "127.7.234.126" port: 34333 } } peers { permanent_uuid: "58a5c5945c8a4f35b49aa5dcbed1c006" member_type: VOTER last_known_addr { host: "127.7.234.125" port: 33623 } } peers { permanent_uuid: "16434748810647be9737fc5cf81d3164" member_type: VOTER last_known_addr { host: "127.7.234.124" port: 38601 } }
02:06:34.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:34.694546 8926 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.234.124:38601: connect: Connection refused (error 111)
02:06:34.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:34.695160 9023 leader_election.cc:290] T 00000000000000000000000000000000 P 6a385942aca04c84b16cd85c5407bae0 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 58a5c5945c8a4f35b49aa5dcbed1c006 (127.7.234.125:33623), 16434748810647be9737fc5cf81d3164 (127.7.234.124:38601)
02:06:34.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:34.712182 8924 leader_election.cc:336] T 00000000000000000000000000000000 P 6a385942aca04c84b16cd85c5407bae0 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 58a5c5945c8a4f35b49aa5dcbed1c006 (127.7.234.125:33623): Network error: Client connection negotiation failed: client connection to 127.7.234.125:33623: connect: Connection refused (error 111)
02:06:34.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:34.716358 8926 leader_election.cc:336] T 00000000000000000000000000000000 P 6a385942aca04c84b16cd85c5407bae0 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 16434748810647be9737fc5cf81d3164 (127.7.234.124:38601): Network error: Client connection negotiation failed: client connection to 127.7.234.124:38601: connect: Connection refused (error 111)
02:06:34.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:34.717000 8926 leader_election.cc:304] T 00000000000000000000000000000000 P 6a385942aca04c84b16cd85c5407bae0 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6a385942aca04c84b16cd85c5407bae0; no voters: 16434748810647be9737fc5cf81d3164, 58a5c5945c8a4f35b49aa5dcbed1c006
02:06:34.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:34.718081 9023 raft_consensus.cc:2747] T 00000000000000000000000000000000 P 6a385942aca04c84b16cd85c5407bae0 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
02:06:35.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 28 02:06:35 dist-test-slave-lq1z krb5kdc[8119](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1751076395, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.234.65@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:06:35.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:35.452288 8464 init.cc:295] Successfully reacquired a new kerberos TGT
02:06:35.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:34.719810 9005 debug-util.cc:398] Leaking SignalData structure 0x7b0800018780 after lost signal to thread 8980
02:06:35.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:35.523365 9005 kernel_stack_watchdog.cc:198] Thread 8980 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:641 for 398ms:
02:06:35.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
02:06:35.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
02:06:35.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
02:06:35.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
02:06:35.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:33.320252 9006 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:35.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:35.524451 8980 thread.cc:640] OpenStack (cloud detector) Time spent creating pthread: real 2.206s user 0.696s sys 1.047s
02:06:35.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:35.524799 8980 thread.cc:606] OpenStack (cloud detector) Time spent starting thread: real 2.206s user 0.696s sys 1.047s
02:06:35.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:35.527200 9009 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:35.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:35.529289 9008 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2208 milliseconds
02:06:35.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:35.529752 8980 server_base.cc:1043] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:06:35.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:35.531190 8980 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:06:35.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:35.534193 8980 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:06:35.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:35.535629 8980 hybrid_clock.cc:648] HybridClock initialized: now 1751076395535554 us; error 71 us; skew 500 ppm
02:06:35.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:35.536717 8980 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:06:35.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 28 02:06:35 dist-test-slave-lq1z krb5kdc[8119](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1751076395, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.234.125@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:06:35.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:35.548513 8980 init.cc:377] Logged in from keytab as kudu/127.7.234.125@KRBTEST.COM (short username kudu)
02:06:35.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:35.553385 8980 webserver.cc:469] Webserver started at http://127.7.234.125:38081/ using document root <none> and password file <none>
02:06:35.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:35.554242 8980 fs_manager.cc:362] Metadata directory not provided
02:06:35.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:35.554440 8980 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:06:35.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:35.563020 8980 fs_manager.cc:714] Time spent opening directory manager: real 0.006s user 0.005s sys 0.000s
02:06:35.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:35.567739 9037 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:06:35.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:35.569103 8980 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.001s sys 0.002s
02:06:35.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:35.569487 8980 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/master-1/data,/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/master-1/wal
02:06:35.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "58a5c5945c8a4f35b49aa5dcbed1c006"
02:06:35.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-28 02:06:18 on dist-test-slave-lq1z"
02:06:35.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:35.571449 8980 fs_report.cc:389] FS layout report
02:06:35.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:06:35.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/master-1/wal
02:06:35.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/master-1/wal
02:06:35.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/master-1/data/data
02:06:35.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:06:35.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:06:35.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:06:35.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:06:35.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:06:35.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:06:35.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:06:35.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:06:35.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:06:35.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:06:35.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:06:35.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:35.627393 8980 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:06:35.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:35.633371 8980 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:06:35.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:35.633885 8980 kserver.cc:163] Server-wide thread pool size limit: 3276
02:06:35.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:35.747192 8980 rpc_server.cc:307] RPC server started. Bound to: 127.7.234.125:33623
02:06:35.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:35.747354 9089 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.7.234.125:33623 every 8 connection(s)
02:06:35.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:35.750108 8980 server_base.cc:1180] Dumped server information to /tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/master-1/data/info.pb
02:06:35.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:35.754429 8105 external_mini_cluster.cc:1413] Started /tmp/dist-test-task8JcgIg/build/tsan/bin/kudu as pid 8980
02:06:35.755 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:06:35.755 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.7.234.124:38601
02:06:35.756 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
02:06:35.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:35.757264 8105 external_mini_cluster.cc:1351] Running /tmp/dist-test-task8JcgIg/build/tsan/bin/kudu
02:06:35.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task8JcgIg/build/tsan/bin/kudu
02:06:35.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/master-2/wal
02:06:35.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/master-2/data
02:06:35.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:06:35.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:06:35.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:06:35.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:06:35.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:06:35.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:06:35.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/master-2/logs
02:06:35.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/master-2/data/info.pb
02:06:35.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:06:35.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:06:35.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:06:35.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:06:35.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:06:35.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:06:35.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:06:35.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:06:35.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:06:35.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:06:35.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:06:35.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:06:35.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.234.124:38601
02:06:35.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.234.124
02:06:35.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=40485
02:06:35.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/krb5kdc/kudu_127.7.234.124.keytab
02:06:35.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.7.234.124
02:06:35.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:06:35.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
02:06:35.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
02:06:35.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.234.84:33745
02:06:35.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:06:35.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:06:35.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:06:35.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:06:35.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.234.126:34333,127.7.234.125:33623,127.7.234.124:38601
02:06:35.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:06:35.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35625/jwks.json
02:06:35.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
02:06:35.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:35.764897 9090 sys_catalog.cc:263] Verifying existing consensus state
02:06:35.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:35.788925 9090 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 58a5c5945c8a4f35b49aa5dcbed1c006: Bootstrap starting.
02:06:35.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:35.884270 9090 log.cc:826] T 00000000000000000000000000000000 P 58a5c5945c8a4f35b49aa5dcbed1c006: Log is configured to *not* fsync() on all Append() calls
02:06:35.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:35.975447 9090 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 58a5c5945c8a4f35b49aa5dcbed1c006: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
02:06:35.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:35.976562 9090 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 58a5c5945c8a4f35b49aa5dcbed1c006: Bootstrap complete.
02:06:35.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:35.977896 8877 heartbeater.cc:344] Connected to a master server at 127.7.234.125:33623
02:06:36.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:36.011314 9090 raft_consensus.cc:357] T 00000000000000000000000000000000 P 58a5c5945c8a4f35b49aa5dcbed1c006 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a385942aca04c84b16cd85c5407bae0" member_type: VOTER last_known_addr { host: "127.7.234.126" port: 34333 } } peers { permanent_uuid: "58a5c5945c8a4f35b49aa5dcbed1c006" member_type: VOTER last_known_addr { host: "127.7.234.125" port: 33623 } } peers { permanent_uuid: "16434748810647be9737fc5cf81d3164" member_type: VOTER last_known_addr { host: "127.7.234.124" port: 38601 } }
02:06:36.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:36.014906 9090 raft_consensus.cc:738] T 00000000000000000000000000000000 P 58a5c5945c8a4f35b49aa5dcbed1c006 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 58a5c5945c8a4f35b49aa5dcbed1c006, State: Initialized, Role: FOLLOWER
02:06:36.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:36.016407 9090 consensus_queue.cc:260] T 00000000000000000000000000000000 P 58a5c5945c8a4f35b49aa5dcbed1c006 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a385942aca04c84b16cd85c5407bae0" member_type: VOTER last_known_addr { host: "127.7.234.126" port: 34333 } } peers { permanent_uuid: "58a5c5945c8a4f35b49aa5dcbed1c006" member_type: VOTER last_known_addr { host: "127.7.234.125" port: 33623 } } peers { permanent_uuid: "16434748810647be9737fc5cf81d3164" member_type: VOTER last_known_addr { host: "127.7.234.124" port: 38601 } }
02:06:36.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:36.020342 9097 sys_catalog.cc:455] T 00000000000000000000000000000000 P 58a5c5945c8a4f35b49aa5dcbed1c006 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a385942aca04c84b16cd85c5407bae0" member_type: VOTER last_known_addr { host: "127.7.234.126" port: 34333 } } peers { permanent_uuid: "58a5c5945c8a4f35b49aa5dcbed1c006" member_type: VOTER last_known_addr { host: "127.7.234.125" port: 33623 } } peers { permanent_uuid: "16434748810647be9737fc5cf81d3164" member_type: VOTER last_known_addr { host: "127.7.234.124" port: 38601 } } }
02:06:36.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:36.021881 9097 sys_catalog.cc:458] T 00000000000000000000000000000000 P 58a5c5945c8a4f35b49aa5dcbed1c006 [sys.catalog]: This master's current role is: FOLLOWER
02:06:36.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:36.022428 9090 sys_catalog.cc:564] T 00000000000000000000000000000000 P 58a5c5945c8a4f35b49aa5dcbed1c006 [sys.catalog]: configured and running, proceeding with master startup.
02:06:36.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:36.073118 9108 catalog_manager.cc:1261] Loaded cluster ID: c5aad08e8e8643a6baa349b9d79cbd4e
02:06:36.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:36.073694 9108 catalog_manager.cc:1554] T 00000000000000000000000000000000 P 58a5c5945c8a4f35b49aa5dcbed1c006: loading cluster ID for follower catalog manager: success
02:06:36.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 28 02:06:36 dist-test-slave-lq1z krb5kdc[8119](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1751076395, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.234.125@KRBTEST.COM for kudu/127.7.234.126@KRBTEST.COM
02:06:36.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:36.091006 9108 catalog_manager.cc:1576] T 00000000000000000000000000000000 P 58a5c5945c8a4f35b49aa5dcbed1c006: acquiring CA information for follower catalog manager: success
02:06:36.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:36.116061 9108 catalog_manager.cc:1604] T 00000000000000000000000000000000 P 58a5c5945c8a4f35b49aa5dcbed1c006: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
02:06:36.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:36.140967 8582 heartbeater.cc:344] Connected to a master server at 127.7.234.125:33623
02:06:36.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:36.148226 9055 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "8e3353a7494549b49fde87660476a705" instance_seqno: 1751076385722956) as {username='kudu', principal='kudu/127.7.234.65@KRBTEST.COM'} at 127.7.234.65:47219; Asking this server to re-register.
02:06:36.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:36.165409 8582 heartbeater.cc:461] Registering TS with master...
02:06:36.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:36.168838 9055 ts_manager.cc:194] Registered new tserver with Master: 8e3353a7494549b49fde87660476a705 (127.7.234.65:38803)
02:06:36.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:36.150475 8980 master_runner.cc:186] Error getting master registration for 127.7.234.124:38601: Network error: Client connection negotiation failed: client connection to 127.7.234.124:38601: connect: Connection refused (error 111),
02:06:36.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:36.209913 8980 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
02:06:36.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:36.277181 9092 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:06:36.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:36.278028 9092 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:06:36.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:36.279186 9092 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:06:36.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:36.341243 9092 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
02:06:36.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:36.341598 9092 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
02:06:36.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:36.342079 9092 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:06:36.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:36.342445 9092 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
02:06:36.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:36.342801 9092 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
02:06:36.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:36.343128 9092 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
02:06:36.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:36.343380 9092 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:35625/jwks.json
02:06:36.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:36.410768 9092 master_runner.cc:386] Master server non-default flags:
02:06:36.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:06:36.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.234.84:33745
02:06:36.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:06:36.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:06:36.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/master-2/data
02:06:36.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/master-2/wal
02:06:36.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
02:06:36.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:06:36.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.234.126:34333,127.7.234.125:33623,127.7.234.124:38601
02:06:36.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/krb5kdc/kudu_127.7.234.124.keytab
02:06:36.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.7.234.124
02:06:36.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:06:36.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:06:36.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:06:36.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.234.124:38601
02:06:36.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:06:36.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:06:36.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:06:36.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35625/jwks.json
02:06:36.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:06:36.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
02:06:36.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
02:06:36.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:06:36.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:06:36.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/master-2/data/info.pb
02:06:36.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.234.124
02:06:36.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=40485
02:06:36.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:06:36.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:06:36.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:06:36.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:06:36.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:06:36.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/master-2/logs
02:06:36.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:06:36.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:06:36.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:06:36.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:06:36.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
02:06:36.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision f0776864a65c19ebdd909053f3e624e8bcc25cee
02:06:36.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:06:36.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 28 Jun 2025 01:43:19 UTC on 5fd53c4cbb9d
02:06:36.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 6805
02:06:36.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:06:36.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:36.413106 9092 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:06:36.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:36.414616 9092 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:06:36.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:36.436854 9121 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:36.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:36.491654 9125 raft_consensus.cc:491] T 00000000000000000000000000000000 P 6a385942aca04c84b16cd85c5407bae0 [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:06:36.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:36.492589 9125 raft_consensus.cc:513] T 00000000000000000000000000000000 P 6a385942aca04c84b16cd85c5407bae0 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a385942aca04c84b16cd85c5407bae0" member_type: VOTER last_known_addr { host: "127.7.234.126" port: 34333 } } peers { permanent_uuid: "58a5c5945c8a4f35b49aa5dcbed1c006" member_type: VOTER last_known_addr { host: "127.7.234.125" port: 33623 } } peers { permanent_uuid: "16434748810647be9737fc5cf81d3164" member_type: VOTER last_known_addr { host: "127.7.234.124" port: 38601 } }
02:06:36.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:36.519989 9125 leader_election.cc:290] T 00000000000000000000000000000000 P 6a385942aca04c84b16cd85c5407bae0 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 58a5c5945c8a4f35b49aa5dcbed1c006 (127.7.234.125:33623), 16434748810647be9737fc5cf81d3164 (127.7.234.124:38601)
02:06:36.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:36.537876 9065 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "6a385942aca04c84b16cd85c5407bae0" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "58a5c5945c8a4f35b49aa5dcbed1c006" is_pre_election: true
02:06:36.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:36.538949 9065 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 58a5c5945c8a4f35b49aa5dcbed1c006 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6a385942aca04c84b16cd85c5407bae0 in term 1.
02:06:36.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:36.545141 8924 leader_election.cc:304] T 00000000000000000000000000000000 P 6a385942aca04c84b16cd85c5407bae0 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 58a5c5945c8a4f35b49aa5dcbed1c006, 6a385942aca04c84b16cd85c5407bae0; no voters:
02:06:36.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:36.547286 9125 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 6a385942aca04c84b16cd85c5407bae0 [term 1 FOLLOWER]: Leader pre-election won for term 2
02:06:36.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:36.547849 9125 raft_consensus.cc:491] T 00000000000000000000000000000000 P 6a385942aca04c84b16cd85c5407bae0 [term 1 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
02:06:36.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:36.548223 9125 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 6a385942aca04c84b16cd85c5407bae0 [term 1 FOLLOWER]: Advancing to term 2
02:06:36.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:36.565313 9125 raft_consensus.cc:513] T 00000000000000000000000000000000 P 6a385942aca04c84b16cd85c5407bae0 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a385942aca04c84b16cd85c5407bae0" member_type: VOTER last_known_addr { host: "127.7.234.126" port: 34333 } } peers { permanent_uuid: "58a5c5945c8a4f35b49aa5dcbed1c006" member_type: VOTER last_known_addr { host: "127.7.234.125" port: 33623 } } peers { permanent_uuid: "16434748810647be9737fc5cf81d3164" member_type: VOTER last_known_addr { host: "127.7.234.124" port: 38601 } }
02:06:36.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:36.568517 9125 leader_election.cc:290] T 00000000000000000000000000000000 P 6a385942aca04c84b16cd85c5407bae0 [CANDIDATE]: Term 2 election: Requested vote from peers 58a5c5945c8a4f35b49aa5dcbed1c006 (127.7.234.125:33623), 16434748810647be9737fc5cf81d3164 (127.7.234.124:38601)
02:06:36.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:36.569152 9065 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "6a385942aca04c84b16cd85c5407bae0" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "58a5c5945c8a4f35b49aa5dcbed1c006"
02:06:36.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:36.569769 9065 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 58a5c5945c8a4f35b49aa5dcbed1c006 [term 1 FOLLOWER]: Advancing to term 2
02:06:36.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:36.580629 8926 leader_election.cc:336] T 00000000000000000000000000000000 P 6a385942aca04c84b16cd85c5407bae0 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 16434748810647be9737fc5cf81d3164 (127.7.234.124:38601): Network error: Client connection negotiation failed: client connection to 127.7.234.124:38601: connect: Connection refused (error 111)
02:06:36.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:36.586259 9065 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 58a5c5945c8a4f35b49aa5dcbed1c006 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6a385942aca04c84b16cd85c5407bae0 in term 2.
02:06:36.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:36.586236 8926 leader_election.cc:336] T 00000000000000000000000000000000 P 6a385942aca04c84b16cd85c5407bae0 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 16434748810647be9737fc5cf81d3164 (127.7.234.124:38601): Network error: Client connection negotiation failed: client connection to 127.7.234.124:38601: connect: Connection refused (error 111)
02:06:36.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:36.587728 8924 leader_election.cc:304] T 00000000000000000000000000000000 P 6a385942aca04c84b16cd85c5407bae0 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 58a5c5945c8a4f35b49aa5dcbed1c006, 6a385942aca04c84b16cd85c5407bae0; no voters: 16434748810647be9737fc5cf81d3164
02:06:36.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:36.589000 9125 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 6a385942aca04c84b16cd85c5407bae0 [term 2 FOLLOWER]: Leader election won for term 2
02:06:36.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:36.613123 9125 raft_consensus.cc:695] T 00000000000000000000000000000000 P 6a385942aca04c84b16cd85c5407bae0 [term 2 LEADER]: Becoming Leader. State: Replica: 6a385942aca04c84b16cd85c5407bae0, State: Running, Role: LEADER
02:06:36.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:36.614471 9125 consensus_queue.cc:237] T 00000000000000000000000000000000 P 6a385942aca04c84b16cd85c5407bae0 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a385942aca04c84b16cd85c5407bae0" member_type: VOTER last_known_addr { host: "127.7.234.126" port: 34333 } } peers { permanent_uuid: "58a5c5945c8a4f35b49aa5dcbed1c006" member_type: VOTER last_known_addr { host: "127.7.234.125" port: 33623 } } peers { permanent_uuid: "16434748810647be9737fc5cf81d3164" member_type: VOTER last_known_addr { host: "127.7.234.124" port: 38601 } }
02:06:36.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:36.649689 8731 heartbeater.cc:344] Connected to a master server at 127.7.234.125:33623
02:06:36.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:36.654598 9130 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6a385942aca04c84b16cd85c5407bae0 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 6a385942aca04c84b16cd85c5407bae0. Latest consensus state: current_term: 2 leader_uuid: "6a385942aca04c84b16cd85c5407bae0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a385942aca04c84b16cd85c5407bae0" member_type: VOTER last_known_addr { host: "127.7.234.126" port: 34333 } } peers { permanent_uuid: "58a5c5945c8a4f35b49aa5dcbed1c006" member_type: VOTER last_known_addr { host: "127.7.234.125" port: 33623 } } peers { permanent_uuid: "16434748810647be9737fc5cf81d3164" member_type: VOTER last_known_addr { host: "127.7.234.124" port: 38601 } } }
02:06:36.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:36.655370 9130 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6a385942aca04c84b16cd85c5407bae0 [sys.catalog]: This master's current role is: LEADER
02:06:36.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:36.662586 9055 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "43991169d0e84efcb4f4b7bca98ff922" instance_seqno: 1751076388428833) as {username='kudu', principal='kudu/127.7.234.66@KRBTEST.COM'} at 127.7.234.66:36619; Asking this server to re-register.
02:06:36.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:36.665129 8731 heartbeater.cc:461] Registering TS with master...
02:06:36.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:36.669003 9055 ts_manager.cc:194] Registered new tserver with Master: 43991169d0e84efcb4f4b7bca98ff922 (127.7.234.66:43479)
02:06:36.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:36.689594 9133 catalog_manager.cc:1477] Loading table and tablet metadata into memory...
02:06:36.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:36.746925 9133 catalog_manager.cc:1486] Initializing Kudu cluster ID...
02:06:36.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:36.770920 9133 catalog_manager.cc:1261] Loaded cluster ID: c5aad08e8e8643a6baa349b9d79cbd4e
02:06:36.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:36.780200 9133 catalog_manager.cc:1497] Initializing Kudu internal certificate authority...
02:06:36.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:36.785943 9133 catalog_manager.cc:1506] Loading token signing keys...
02:06:36.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:36.796370 9133 catalog_manager.cc:5966] T 00000000000000000000000000000000 P 6a385942aca04c84b16cd85c5407bae0: Loaded TSK: 0
02:06:36.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:36.801716 9133 catalog_manager.cc:1516] Initializing in-progress tserver states...
02:06:36.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:36.993043 9055 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "fb8d9e2f1452456e88e19234cb1b835f" instance_seqno: 1751076390602995) as {username='kudu', principal='kudu/127.7.234.67@KRBTEST.COM'} at 127.7.234.67:34275; Asking this server to re-register.
02:06:37.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:37.004248 8877 heartbeater.cc:461] Registering TS with master...
02:06:37.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:37.015892 9055 ts_manager.cc:194] Registered new tserver with Master: fb8d9e2f1452456e88e19234cb1b835f (127.7.234.67:45767)
02:06:37.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:37.059196 7562 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.7.52.254:46311, user_credentials={real_user=slave}} blocked reactor thread for 56688.4us
02:06:37.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:37.080183 9065 raft_consensus.cc:1273] T 00000000000000000000000000000000 P 58a5c5945c8a4f35b49aa5dcbed1c006 [term 2 FOLLOWER]: Refusing update from remote peer 6a385942aca04c84b16cd85c5407bae0: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 2 index: 5. (index mismatch)
02:06:37.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:37.082331 9130 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 6a385942aca04c84b16cd85c5407bae0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "58a5c5945c8a4f35b49aa5dcbed1c006" member_type: VOTER last_known_addr { host: "127.7.234.125" port: 33623 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.001s
02:06:37.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:37.190050 9136 sys_catalog.cc:455] T 00000000000000000000000000000000 P 58a5c5945c8a4f35b49aa5dcbed1c006 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 6a385942aca04c84b16cd85c5407bae0. Latest consensus state: current_term: 2 leader_uuid: "6a385942aca04c84b16cd85c5407bae0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a385942aca04c84b16cd85c5407bae0" member_type: VOTER last_known_addr { host: "127.7.234.126" port: 34333 } } peers { permanent_uuid: "58a5c5945c8a4f35b49aa5dcbed1c006" member_type: VOTER last_known_addr { host: "127.7.234.125" port: 33623 } } peers { permanent_uuid: "16434748810647be9737fc5cf81d3164" member_type: VOTER last_known_addr { host: "127.7.234.124" port: 38601 } } }
02:06:37.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:37.190860 9136 sys_catalog.cc:458] T 00000000000000000000000000000000 P 58a5c5945c8a4f35b49aa5dcbed1c006 [sys.catalog]: This master's current role is: FOLLOWER
02:06:37.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:37.233314 9130 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6a385942aca04c84b16cd85c5407bae0 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "6a385942aca04c84b16cd85c5407bae0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a385942aca04c84b16cd85c5407bae0" member_type: VOTER last_known_addr { host: "127.7.234.126" port: 34333 } } peers { permanent_uuid: "58a5c5945c8a4f35b49aa5dcbed1c006" member_type: VOTER last_known_addr { host: "127.7.234.125" port: 33623 } } peers { permanent_uuid: "16434748810647be9737fc5cf81d3164" member_type: VOTER last_known_addr { host: "127.7.234.124" port: 38601 } } }
02:06:37.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:37.235580 9130 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6a385942aca04c84b16cd85c5407bae0 [sys.catalog]: This master's current role is: LEADER
02:06:37.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:37.242030 9136 sys_catalog.cc:455] T 00000000000000000000000000000000 P 58a5c5945c8a4f35b49aa5dcbed1c006 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "6a385942aca04c84b16cd85c5407bae0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a385942aca04c84b16cd85c5407bae0" member_type: VOTER last_known_addr { host: "127.7.234.126" port: 34333 } } peers { permanent_uuid: "58a5c5945c8a4f35b49aa5dcbed1c006" member_type: VOTER last_known_addr { host: "127.7.234.125" port: 33623 } } peers { permanent_uuid: "16434748810647be9737fc5cf81d3164" member_type: VOTER last_known_addr { host: "127.7.234.124" port: 38601 } } }
02:06:37.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:37.242997 9136 sys_catalog.cc:458] T 00000000000000000000000000000000 P 58a5c5945c8a4f35b49aa5dcbed1c006 [sys.catalog]: This master's current role is: FOLLOWER
02:06:37.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:37.258472 8926 consensus_peers.cc:489] T 00000000000000000000000000000000 P 6a385942aca04c84b16cd85c5407bae0 -> Peer 16434748810647be9737fc5cf81d3164 (127.7.234.124:38601): Couldn't send request to peer 16434748810647be9737fc5cf81d3164. Status: Network error: Client connection negotiation failed: client connection to 127.7.234.124:38601: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
02:06:37.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:37.855530 8880 heartbeater.cc:499] Master 127.7.234.126:34333 was elected leader, sending a full tablet report...
02:06:38.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:38.058362 8872 debug-util.cc:398] Leaking SignalData structure 0x7b08000ba2a0 after lost signal to thread 8745
02:06:38.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:38.059413 8872 debug-util.cc:398] Leaking SignalData structure 0x7b08000bae80 after lost signal to thread 8875
02:06:38.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:38.089483 8891 master_runner.cc:186] Error getting master registration for 127.7.234.124:38601: Network error: Client connection negotiation failed: client connection to 127.7.234.124:38601: connect: Connection refused (error 111),
02:06:38.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:38.089996 8891 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
02:06:38.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:38.105458 8585 heartbeater.cc:499] Master 127.7.234.126:34333 was elected leader, sending a full tablet report...
02:06:38.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:37.831019 9120 debug-util.cc:398] Leaking SignalData structure 0x7b0800018780 after lost signal to thread 9092
02:06:38.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:38.130172 9092 thread.cc:640] GCE (cloud detector) Time spent creating pthread: real 1.694s user 0.604s sys 0.843s
02:06:38.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:38.130626 9092 thread.cc:606] GCE (cloud detector) Time spent starting thread: real 1.695s user 0.604s sys 0.843s
02:06:38.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:36.438298 9122 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:38.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:38.145304 9148 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:06:38.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:38.160632 9092 server_base.cc:1048] running on GCE node
02:06:38.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:38.163486 9092 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:06:38.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:38.167245 9092 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:06:38.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:38.175392 9092 hybrid_clock.cc:648] HybridClock initialized: now 1751076398175206 us; error 131 us; skew 500 ppm
02:06:38.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:38.177018 9092 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:06:38.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 28 02:06:38 dist-test-slave-lq1z krb5kdc[8119](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1751076398, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.234.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:06:38.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:38.191798 9092 init.cc:377] Logged in from keytab as kudu/127.7.234.124@KRBTEST.COM (short username kudu)
02:06:38.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 28 02:06:38 dist-test-slave-lq1z krb5kdc[8119](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1751076398, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.234.66@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:06:38.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:38.215145 8611 init.cc:295] Successfully reacquired a new kerberos TGT
02:06:38.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:38.237396 9092 webserver.cc:469] Webserver started at http://127.7.234.124:40485/ using document root <none> and password file <none>
02:06:38.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:38.238883 9092 fs_manager.cc:362] Metadata directory not provided
02:06:38.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:38.239219 9092 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:06:38.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:38.261869 9092 fs_manager.cc:714] Time spent opening directory manager: real 0.016s user 0.006s sys 0.006s
02:06:38.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:38.274936 9156 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:06:38.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:38.276968 9092 fs_manager.cc:730] Time spent opening block manager: real 0.012s user 0.001s sys 0.008s
02:06:38.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:38.277622 9092 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/master-2/data,/tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/master-2/wal
02:06:38.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "16434748810647be9737fc5cf81d3164"
02:06:38.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-28 02:06:20 on dist-test-slave-lq1z"
02:06:38.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:38.284786 9092 fs_report.cc:389] FS layout report
02:06:38.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:06:38.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/master-2/wal
02:06:38.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/master-2/wal
02:06:38.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/master-2/data/data
02:06:38.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:06:38.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:06:38.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:06:38.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:06:38.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:06:38.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:06:38.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:06:38.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:06:38.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:06:38.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:06:38.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:06:38.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:38.381527 9092 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:06:38.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:38.392087 9092 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:06:38.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:38.393064 9092 kserver.cc:163] Server-wide thread pool size limit: 3276
02:06:38.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:38.576359 9092 rpc_server.cc:307] RPC server started. Bound to: 127.7.234.124:38601
02:06:38.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:38.576345 9208 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.7.234.124:38601 every 8 connection(s)
02:06:38.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:38.581314 9092 server_base.cc:1180] Dumped server information to /tmp/dist-test-task8JcgIg/test-tmp/mini-kudu-cluster9032905556771096809/master-2/data/info.pb
02:06:38.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:38.584028 8105 external_mini_cluster.cc:1413] Started /tmp/dist-test-task8JcgIg/build/tsan/bin/kudu as pid 9092
02:06:38.588 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:06:38.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:38.600821 9209 sys_catalog.cc:263] Verifying existing consensus state
02:06:38.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:38.611644 9209 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 16434748810647be9737fc5cf81d3164: Bootstrap starting.
02:06:38.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:38.689023 8732 heartbeater.cc:499] Master 127.7.234.126:34333 was elected leader, sending a full tablet report...
02:06:38.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:38.707604 9209 log.cc:826] T 00000000000000000000000000000000 P 16434748810647be9737fc5cf81d3164: Log is configured to *not* fsync() on all Append() calls
02:06:38.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:38.773810 9209 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 16434748810647be9737fc5cf81d3164: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
02:06:38.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:38.774964 9209 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 16434748810647be9737fc5cf81d3164: Bootstrap complete.
02:06:38.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 28 02:06:38 dist-test-slave-lq1z krb5kdc[8119](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1751076398, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.234.66@KRBTEST.COM for kudu/127.7.234.124@KRBTEST.COM
02:06:38.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:38.813535 8730 heartbeater.cc:344] Connected to a master server at 127.7.234.124:38601
02:06:38.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 28 02:06:38 dist-test-slave-lq1z krb5kdc[8119](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1751076392, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.234.126@KRBTEST.COM for kudu/127.7.234.124@KRBTEST.COM
02:06:38.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:38.835935 9209 raft_consensus.cc:357] T 00000000000000000000000000000000 P 16434748810647be9737fc5cf81d3164 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a385942aca04c84b16cd85c5407bae0" member_type: VOTER last_known_addr { host: "127.7.234.126" port: 34333 } } peers { permanent_uuid: "58a5c5945c8a4f35b49aa5dcbed1c006" member_type: VOTER last_known_addr { host: "127.7.234.125" port: 33623 } } peers { permanent_uuid: "16434748810647be9737fc5cf81d3164" member_type: VOTER last_known_addr { host: "127.7.234.124" port: 38601 } }
02:06:38.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:38.850407 9209 raft_consensus.cc:738] T 00000000000000000000000000000000 P 16434748810647be9737fc5cf81d3164 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 16434748810647be9737fc5cf81d3164, State: Initialized, Role: FOLLOWER
02:06:38.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:38.851977 9209 consensus_queue.cc:260] T 00000000000000000000000000000000 P 16434748810647be9737fc5cf81d3164 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a385942aca04c84b16cd85c5407bae0" member_type: VOTER last_known_addr { host: "127.7.234.126" port: 34333 } } peers { permanent_uuid: "58a5c5945c8a4f35b49aa5dcbed1c006" member_type: VOTER last_known_addr { host: "127.7.234.125" port: 33623 } } peers { permanent_uuid: "16434748810647be9737fc5cf81d3164" member_type: VOTER last_known_addr { host: "127.7.234.124" port: 38601 } }
02:06:38.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:38.873167 9209 sys_catalog.cc:564] T 00000000000000000000000000000000 P 16434748810647be9737fc5cf81d3164 [sys.catalog]: configured and running, proceeding with master startup.
02:06:38.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:38.879017 9184 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 16434748810647be9737fc5cf81d3164 [term 1 FOLLOWER]: Advancing to term 2
02:06:38.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:38.888236 9217 sys_catalog.cc:455] T 00000000000000000000000000000000 P 16434748810647be9737fc5cf81d3164 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a385942aca04c84b16cd85c5407bae0" member_type: VOTER last_known_addr { host: "127.7.234.126" port: 34333 } } peers { permanent_uuid: "58a5c5945c8a4f35b49aa5dcbed1c006" member_type: VOTER last_known_addr { host: "127.7.234.125" port: 33623 } } peers { permanent_uuid: "16434748810647be9737fc5cf81d3164" member_type: VOTER last_known_addr { host: "127.7.234.124" port: 38601 } } }
02:06:38.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:38.891249 9217 sys_catalog.cc:458] T 00000000000000000000000000000000 P 16434748810647be9737fc5cf81d3164 [sys.catalog]: This master's current role is: FOLLOWER
02:06:38.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:38.939589 9217 sys_catalog.cc:455] T 00000000000000000000000000000000 P 16434748810647be9737fc5cf81d3164 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 6a385942aca04c84b16cd85c5407bae0. Latest consensus state: current_term: 2 leader_uuid: "6a385942aca04c84b16cd85c5407bae0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a385942aca04c84b16cd85c5407bae0" member_type: VOTER last_known_addr { host: "127.7.234.126" port: 34333 } } peers { permanent_uuid: "58a5c5945c8a4f35b49aa5dcbed1c006" member_type: VOTER last_known_addr { host: "127.7.234.125" port: 33623 } } peers { permanent_uuid: "16434748810647be9737fc5cf81d3164" member_type: VOTER last_known_addr { host: "127.7.234.124" port: 38601 } } }
02:06:38.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:38.941810 9217 sys_catalog.cc:458] T 00000000000000000000000000000000 P 16434748810647be9737fc5cf81d3164 [sys.catalog]: This master's current role is: FOLLOWER
02:06:38.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:38.949082 9224 sys_catalog.cc:455] T 00000000000000000000000000000000 P 16434748810647be9737fc5cf81d3164 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "6a385942aca04c84b16cd85c5407bae0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a385942aca04c84b16cd85c5407bae0" member_type: VOTER last_known_addr { host: "127.7.234.126" port: 34333 } } peers { permanent_uuid: "58a5c5945c8a4f35b49aa5dcbed1c006" member_type: VOTER last_known_addr { host: "127.7.234.125" port: 33623 } } peers { permanent_uuid: "16434748810647be9737fc5cf81d3164" member_type: VOTER last_known_addr { host: "127.7.234.124" port: 38601 } } }
02:06:38.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:38.950322 9224 sys_catalog.cc:458] T 00000000000000000000000000000000 P 16434748810647be9737fc5cf81d3164 [sys.catalog]: This master's current role is: FOLLOWER
02:06:38.955 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.7.234.126:34333(127.7.234.126:34333)] connecting to peer
02:06:39.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:39.060899 8876 heartbeater.cc:344] Connected to a master server at 127.7.234.124:38601
02:06:39.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:39.061784 8876 heartbeater.cc:499] Master 127.7.234.124:38601 was elected leader, sending a full tablet report...
02:06:39.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:39.068198 9174 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "fb8d9e2f1452456e88e19234cb1b835f" instance_seqno: 1751076390602995) as {username='kudu', principal='kudu/127.7.234.67@KRBTEST.COM'} at 127.7.234.67:60951; Asking this server to re-register.
02:06:39.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:39.071105 8876 heartbeater.cc:461] Registering TS with master...
02:06:39.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:39.072719 9233 catalog_manager.cc:1261] Loaded cluster ID: c5aad08e8e8643a6baa349b9d79cbd4e
02:06:39.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:39.081668 9233 catalog_manager.cc:1554] T 00000000000000000000000000000000 P 16434748810647be9737fc5cf81d3164: loading cluster ID for follower catalog manager: success
02:06:39.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:39.074941 9174 ts_manager.cc:194] Registered new tserver with Master: fb8d9e2f1452456e88e19234cb1b835f (127.7.234.67:45767)
02:06:39.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:39.097896 9233 catalog_manager.cc:1576] T 00000000000000000000000000000000 P 16434748810647be9737fc5cf81d3164: acquiring CA information for follower catalog manager: success
02:06:39.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 28 02:06:39 dist-test-slave-lq1z krb5kdc[8119](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1751076398, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.234.124@KRBTEST.COM for kudu/127.7.234.126@KRBTEST.COM
02:06:39.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:39.116184 9233 catalog_manager.cc:1604] T 00000000000000000000000000000000 P 16434748810647be9737fc5cf81d3164: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
02:06:39.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:39.230216 8581 heartbeater.cc:344] Connected to a master server at 127.7.234.124:38601
02:06:39.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:39.233714 9174 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "8e3353a7494549b49fde87660476a705" instance_seqno: 1751076385722956) as {username='kudu', principal='kudu/127.7.234.65@KRBTEST.COM'} at 127.7.234.65:47427; Asking this server to re-register.
02:06:39.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:39.235252 8581 heartbeater.cc:461] Registering TS with master...
02:06:39.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:39.239018 9174 ts_manager.cc:194] Registered new tserver with Master: 8e3353a7494549b49fde87660476a705 (127.7.234.65:38803)
02:06:39.250 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.7.234.125:33623(127.7.234.125:33623)] connecting to peer
02:06:39.255 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.7.234.124:38601(127.7.234.124:38601)] connecting to peer
02:06:39.367 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.7.234.125:33623(127.7.234.125:33623)] Successfully connected to peer
02:06:39.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:39.370538 9242 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6a385942aca04c84b16cd85c5407bae0 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "6a385942aca04c84b16cd85c5407bae0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6a385942aca04c84b16cd85c5407bae0" member_type: VOTER last_known_addr { host: "127.7.234.126" port: 34333 } } peers { permanent_uuid: "58a5c5945c8a4f35b49aa5dcbed1c006" member_type: VOTER last_known_addr { host: "127.7.234.125" port: 33623 } } peers { permanent_uuid: "16434748810647be9737fc5cf81d3164" member_type: VOTER last_known_addr { host: "127.7.234.124" port: 38601 } } }
02:06:39.372 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.7.234.126:34333(127.7.234.126:34333)] Successfully connected to peer
02:06:39.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:39.373648 9242 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6a385942aca04c84b16cd85c5407bae0 [sys.catalog]: This master's current role is: LEADER
02:06:39.523 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.7.234.124:38601(127.7.234.124:38601)] Successfully connected to peer
02:06:39.566 [WARN - kudu-nio-1] (SecurityContext.java:206) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
02:06:39.603 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.7.234.126
02:06:39.604 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.7.234.125
02:06:39.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:39.820731 9174 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "43991169d0e84efcb4f4b7bca98ff922" instance_seqno: 1751076388428833) as {username='kudu', principal='kudu/127.7.234.66@KRBTEST.COM'} at 127.7.234.66:53541; Asking this server to re-register.
02:06:39.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:39.822140 8730 heartbeater.cc:461] Registering TS with master...
02:06:39.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:39.826210 9174 ts_manager.cc:194] Registered new tserver with Master: 43991169d0e84efcb4f4b7bca98ff922 (127.7.234.66:43479)
02:06:40.195 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.234.126:34333(127.7.234.126:34333)] cleaning up while in state NEGOTIATING due to: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
02:06:40.197 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.234.126:34333: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
02:06:40.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:40.201258 9234 negotiation.cc:337] Failed RPC negotiation. Trace:
02:06:40.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 02:06:39.360080 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:43116 (local address 127.7.234.126:34333)
02:06:40.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 02:06:39.360882 (+ 802us) server_negotiation.cc:206] Beginning negotiation
02:06:40.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 02:06:39.360913 (+ 31us) server_negotiation.cc:399] Waiting for connection header
02:06:40.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 02:06:39.426449 (+ 65536us) server_negotiation.cc:407] Connection header received
02:06:40.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 02:06:39.506411 (+ 79962us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
02:06:40.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 02:06:39.506476 (+ 65us) server_negotiation.cc:461] Received NEGOTIATE request from client
02:06:40.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 02:06:39.506724 (+ 248us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
02:06:40.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 02:06:39.506894 (+ 170us) server_negotiation.cc:226] Negotiated authn=SASL
02:06:40.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 02:06:40.029767 (+522873us) server_negotiation.cc:365] Received TLS_HANDSHAKE NegotiatePB request
02:06:40.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 02:06:40.031548 (+ 1781us) server_negotiation.cc:377] Sending TLS_HANDSHAKE NegotiatePB response
02:06:40.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 02:06:40.184370 (+152822us) server_negotiation.cc:365] Received TLS_HANDSHAKE NegotiatePB request
02:06:40.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 02:06:40.184928 (+ 558us) server_negotiation.cc:662] Negotiated TLSv1.3 with cipher TLS_AES_128_GCM_SHA256 TLSv1.3 Kx=any Au=any Enc=AESGCM(128) Mac=AEAD
02:06:40.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 02:06:40.199805 (+ 14877us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:43116: BlockingRecv error: failed to read from TLS socket (remote: 127.0.0.1:43116): Cannot send after transport endpoint shutdown (error 108)
02:06:40.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":122}
02:06:40.202 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.234.124:38601(127.7.234.124:38601)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
02:06:40.204 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.234.124:38601: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
02:06:40.206 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.7.234.126:34333(127.7.234.126:34333)] handling channelInactive
02:06:40.206 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.7.234.124:38601(127.7.234.124:38601)] handling channelInactive
02:06:40.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:40.206310 9215 negotiation.cc:337] Failed RPC negotiation. Trace:
02:06:40.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 02:06:39.366850 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:40190 (local address 127.7.234.124:38601)
02:06:40.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 02:06:39.367535 (+ 685us) server_negotiation.cc:206] Beginning negotiation
02:06:40.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 02:06:39.367564 (+ 29us) server_negotiation.cc:399] Waiting for connection header
02:06:40.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 02:06:39.525110 (+157546us) server_negotiation.cc:407] Connection header received
02:06:40.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 02:06:39.525938 (+ 828us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
02:06:40.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 02:06:39.525969 (+ 31us) server_negotiation.cc:461] Received NEGOTIATE request from client
02:06:40.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 02:06:39.526203 (+ 234us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
02:06:40.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 02:06:39.526390 (+ 187us) server_negotiation.cc:226] Negotiated authn=SASL
02:06:40.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 02:06:40.205065 (+678675us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:40190: BlockingRecv error: recv got EOF from 127.0.0.1:40190 (error 108)
02:06:40.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":100}
02:06:40.213 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.234.125:33623(127.7.234.125:33623)] cleaning up while in state NEGOTIATING due to: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
02:06:40.214 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.234.125:33623: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
02:06:40.222 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1523) Notify RPC KuduRpc(method=ListTabletServers, tablet=null, attempt=1, TimeoutTracker(timeout=30000, elapsed=1628), Traces: [0ms] refreshing cache from master, [183ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.234.126:34333, [640ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.234.125:33623, [645ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.234.124:38601, [1586ms] Sub RPC ConnectToMaster: received response from server master-127.7.234.126:34333: Configuration error: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), [1593ms] Sub RPC ConnectToMaster: received response from server master-127.7.234.124:38601: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [1603ms] Sub RPC ConnectToMaster: received response from server master-127.7.234.125:33623: Configuration error: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), deferred=null) after lookup exception
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.7.234.126:34333,127.7.234.125:33623,127.7.234.124:38601). Exceptions received: [org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268) [main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) [main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387) [main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376) [main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) [async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) [async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315) [async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286) [async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) [async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) [async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) [main/:?]
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331) [main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256) [main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) [main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) [main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) [main/:?]
at org.apache.kudu.client.Connection.cleanup(Connection.java:737) [main/:?]
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455) [main/:?]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) [netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) [netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) [netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) [netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) [netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_275]
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.7.234.126:34333,127.7.234.125:33623,127.7.234.124:38601). Exceptions received: [org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
02:06:40.227 [DEBUG - kudu-nio-1] (KuduRpc.java:260) Handling a callback on RPC KuduRpc(method=ListTabletServers, tablet=null, attempt=1, TimeoutTracker(timeout=30000, elapsed=1633), Traces: [0ms] refreshing cache from master, [183ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.234.126:34333, [640ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.234.125:33623, [645ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.234.124:38601, [1586ms] Sub RPC ConnectToMaster: received response from server master-127.7.234.126:34333: Configuration error: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), [1593ms] Sub RPC ConnectToMaster: received response from server master-127.7.234.124:38601: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [1603ms] Sub RPC ConnectToMaster: received response from server master-127.7.234.125:33623: Configuration error: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), deferred=null) with no deferred attached!
02:06:40.230 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.7.234.126:34333(127.7.234.126:34333)] disconnecting while in state TERMINATED
02:06:40.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 02:06:40.230753 9237 negotiation.cc:337] Failed RPC negotiation. Trace:
02:06:40.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 02:06:39.357765 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:49306 (local address 127.7.234.125:33623)
02:06:40.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 02:06:39.358477 (+ 712us) server_negotiation.cc:206] Beginning negotiation
02:06:40.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 02:06:39.358506 (+ 29us) server_negotiation.cc:399] Waiting for connection header
02:06:40.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 02:06:39.426821 (+ 68315us) server_negotiation.cc:407] Connection header received
02:06:40.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 02:06:39.503494 (+ 76673us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
02:06:40.231 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.7.234.125:33623(127.7.234.125:33623)] disconnecting while in state TERMINATED
02:06:40.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 02:06:39.503538 (+ 44us) server_negotiation.cc:461] Received NEGOTIATE request from client
02:06:40.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 02:06:39.503836 (+ 298us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
02:06:40.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 02:06:39.504002 (+ 166us) server_negotiation.cc:226] Negotiated authn=SASL
02:06:40.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 02:06:40.010214 (+506212us) server_negotiation.cc:365] Received TLS_HANDSHAKE NegotiatePB request
02:06:40.231 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.7.234.124:38601(127.7.234.124:38601)] disconnecting while in state TERMINATED
02:06:40.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 02:06:40.011835 (+ 1621us) server_negotiation.cc:377] Sending TLS_HANDSHAKE NegotiatePB response
02:06:40.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 02:06:40.195573 (+183738us) server_negotiation.cc:365] Received TLS_HANDSHAKE NegotiatePB request
02:06:40.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 02:06:40.196184 (+ 611us) server_negotiation.cc:662] Negotiated TLSv1.3 with cipher TLS_AES_128_GCM_SHA256 TLSv1.3 Kx=any Au=any Enc=AESGCM(128) Mac=AEAD
02:06:40.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 02:06:40.229316 (+ 33132us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:49306: BlockingRecv error: failed to read from TLS socket (remote: 127.0.0.1:49306): Cannot send after transport endpoint shutdown (error 108)
02:06:40.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":115}
02:06:40.245 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2704) Releasing all remaining resources
02:06:40.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 28 02:06:40 dist-test-slave-lq1z krb5kdc[8119](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1751076400, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.234.67@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:06:40.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:40.247980 8758 init.cc:295] Successfully reacquired a new kerberos TGT
02:06:40.253 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.7.234.125:33623(127.7.234.125:33623)] handling channelInactive
02:06:40.253 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
02:06:40.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:40.260927 8105 external_mini_cluster.cc:1620] Killing /tmp/dist-test-task8JcgIg/build/tsan/bin/kudu with pid 8421
02:06:40.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:40.288523 8105 external_mini_cluster.cc:1620] Killing /tmp/dist-test-task8JcgIg/build/tsan/bin/kudu with pid 8593
02:06:40.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:40.324585 8105 external_mini_cluster.cc:1620] Killing /tmp/dist-test-task8JcgIg/build/tsan/bin/kudu with pid 8742
02:06:40.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:40.360266 8105 external_mini_cluster.cc:1620] Killing /tmp/dist-test-task8JcgIg/build/tsan/bin/kudu with pid 8891
02:06:40.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:40.392313 8105 external_mini_cluster.cc:1620] Killing /tmp/dist-test-task8JcgIg/build/tsan/bin/kudu with pid 8980
02:06:40.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:40.422010 8105 external_mini_cluster.cc:1620] Killing /tmp/dist-test-task8JcgIg/build/tsan/bin/kudu with pid 9092
02:06:41.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-06-28T02:06:41Z chronyd exiting
02:06:42.645 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
02:06:42.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:42.646613 7379 external_mini_cluster.cc:1620] Killing /tmp/dist-test-task8JcgIg/build/tsan/bin/kudu with pid 7659
02:06:42.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:42.670671 7379 external_mini_cluster.cc:1620] Killing /tmp/dist-test-task8JcgIg/build/tsan/bin/kudu with pid 7819
02:06:42.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:42.702641 7379 external_mini_cluster.cc:1620] Killing /tmp/dist-test-task8JcgIg/build/tsan/bin/kudu with pid 7962
02:06:42.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:42.735029 7379 external_mini_cluster.cc:1620] Killing /tmp/dist-test-task8JcgIg/build/tsan/bin/kudu with pid 7391
02:06:42.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:42.764556 7379 external_mini_cluster.cc:1620] Killing /tmp/dist-test-task8JcgIg/build/tsan/bin/kudu with pid 7461
02:06:42.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 02:06:42.792707 7379 external_mini_cluster.cc:1620] Killing /tmp/dist-test-task8JcgIg/build/tsan/bin/kudu with pid 7536
02:06:43.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-06-28T02:06:43Z chronyd exiting
02:06:44.942 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.7.234.126:34333,127.7.234.125:33623,127.7.234.124:38601). Exceptions received: [org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)]
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.cleanup(Connection.java:737) ~[main/:?]
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455) ~[main/:?]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.7.234.126:34333,127.7.234.125:33623,127.7.234.124:38601). Exceptions received: [org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)]
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:748)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more