14:05:39.912 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
14:05:39.924 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-task50Kl3p/test-tmp
14:05:39.928 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
14:05:39.932 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-task50Kl3p/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
14:05:40.239 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster742452460306040201"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
14:05:40.375 [DEBUG - main] (MiniKuduCluster.java:195) Response:
14:05:40.376 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
14:05:40.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:40.379699 987 mini_oidc.cc:150] Starting JWKS server
14:05:40.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:40.383387 987 webserver.cc:480] Webserver started at http://127.0.0.1:44417/ using document root <none> and password file <none>
14:05:40.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:40.383832 987 mini_oidc.cc:186] Starting OIDC Discovery server
14:05:40.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:40.385540 987 webserver.cc:480] Webserver started at http://127.0.0.1:39423/ using document root <none> and password file <none>
14:05:40.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-09-02T14:05:40Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
14:05:40.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-09-02T14:05:40Z Disabled control of system clock
14:05:40.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:40.447436 987 external_mini_cluster.cc:1366] Running /tmp/dist-test-task50Kl3p/build/tsan/bin/kudu
14:05:40.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task50Kl3p/build/tsan/bin/kudu
14:05:40.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster742452460306040201/master-0/wal
14:05:40.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster742452460306040201/master-0/data
14:05:40.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:05:40.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:05:40.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:05:40.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:05:40.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:05:40.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:05:40.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster742452460306040201/master-0/logs
14:05:40.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster742452460306040201/master-0/data/info.pb
14:05:40.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:05:40.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:05:40.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:05:40.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:05:40.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:05:40.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:05:40.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:05:40.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:05:40.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:05:40.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:05:40.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:05:40.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:05:40.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.246.254:38851
14:05:40.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.246.254
14:05:40.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:05:40.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.246.212:41037
14:05:40.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:05:40.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:05:40.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:05:40.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:05:40.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.246.254:38851,127.0.246.253:41237,127.0.246.252:36319
14:05:40.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:05:40.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44417/jwks.json with env {}
14:05:40.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:05:40.755352 999 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:05:40.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:05:40.755931 999 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:05:40.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:05:40.756497 999 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:05:40.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:05:40.788618 999 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:05:40.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:05:40.788923 999 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:05:40.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:05:40.789117 999 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:05:40.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:05:40.789350 999 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:05:40.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:05:40.789595 999 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:05:40.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:05:40.789741 999 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44417/jwks.json
14:05:40.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:40.825842 999 master_runner.cc:386] Master server non-default flags:
14:05:40.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:05:40.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.246.212:41037
14:05:40.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:05:40.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:05:40.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster742452460306040201/master-0/data
14:05:40.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster742452460306040201/master-0/wal
14:05:40.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:05:40.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.246.254:38851,127.0.246.253:41237,127.0.246.252:36319
14:05:40.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:05:40.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:05:40.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:05:40.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.246.254:38851
14:05:40.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:05:40.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:05:40.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:05:40.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44417/jwks.json
14:05:40.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:05:40.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:05:40.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster742452460306040201/master-0/data/info.pb
14:05:40.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.246.254
14:05:40.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:05:40.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:05:40.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:05:40.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:05:40.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:05:40.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:05:40.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster742452460306040201/master-0/logs
14:05:40.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:05:40.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:05:40.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:05:40.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:05:40.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:05:40.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 06466b25e294b4d7d455e6c56263318dc34a80ef
14:05:40.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:05:40.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Sep 2025 13:43:21 UTC on 5fd53c4cbb9d
14:05:40.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7865
14:05:40.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:05:40.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:40.827513 999 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:05:40.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:40.829818 999 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:05:40.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:05:40.844973 1005 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:05:42.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:05:40.845103 1006 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:05:42.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:05:40.853765 1008 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:05:42.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:05:42.055889 1007 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
14:05:42.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:42.056030 999 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:05:42.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:42.057700 999 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:05:42.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:42.061465 999 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:05:42.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:42.062943 999 hybrid_clock.cc:648] HybridClock initialized: now 1756821942062891 us; error 71 us; skew 500 ppm
14:05:42.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:42.063894 999 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:05:42.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:42.070719 999 webserver.cc:480] Webserver started at http://127.0.246.254:34151/ using document root <none> and password file <none>
14:05:42.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:42.071928 999 fs_manager.cc:362] Metadata directory not provided
14:05:42.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:42.072175 999 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:05:42.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:42.072739 999 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
14:05:42.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:42.077522 999 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster742452460306040201/master-0/data/instance:
14:05:42.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0c91631a22d842b8b15ae34f9e3fb04c"
14:05:42.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 14:05:42 on dist-test-slave-m0vh"
14:05:42.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:42.078987 999 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster742452460306040201/master-0/wal/instance:
14:05:42.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0c91631a22d842b8b15ae34f9e3fb04c"
14:05:42.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 14:05:42 on dist-test-slave-m0vh"
14:05:42.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:42.087818 999 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.007s sys 0.001s
14:05:42.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:42.093899 1015 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:05:42.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:42.095175 999 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.006s sys 0.000s
14:05:42.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:42.095657 999 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster742452460306040201/master-0/data,/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster742452460306040201/master-0/wal
14:05:42.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0c91631a22d842b8b15ae34f9e3fb04c"
14:05:42.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 14:05:42 on dist-test-slave-m0vh"
14:05:42.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:42.096071 999 fs_report.cc:389] FS layout report
14:05:42.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:05:42.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster742452460306040201/master-0/wal
14:05:42.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster742452460306040201/master-0/wal
14:05:42.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster742452460306040201/master-0/data/data
14:05:42.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:05:42.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:05:42.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:05:42.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:05:42.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:05:42.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:05:42.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:05:42.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:05:42.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:05:42.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:05:42.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:05:42.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:42.149524 999 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:05:42.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:42.151188 999 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:05:42.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:42.151736 999 kserver.cc:163] Server-wide thread pool size limit: 3276
14:05:42.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:42.222294 999 rpc_server.cc:307] RPC server started. Bound to: 127.0.246.254:38851
14:05:42.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:42.222379 1066 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.0.246.254:38851 every 8 connection(s)
14:05:42.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:42.225289 999 server_base.cc:1179] Dumped server information to /tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster742452460306040201/master-0/data/info.pb
14:05:42.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:42.229089 987 external_mini_cluster.cc:1428] Started /tmp/dist-test-task50Kl3p/build/tsan/bin/kudu as pid 999
14:05:42.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:42.229633 987 external_mini_cluster.cc:1442] Reading /tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster742452460306040201/master-0/wal/instance
14:05:42.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:42.231525 1067 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
14:05:42.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:42.234892 987 external_mini_cluster.cc:1366] Running /tmp/dist-test-task50Kl3p/build/tsan/bin/kudu
14:05:42.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task50Kl3p/build/tsan/bin/kudu
14:05:42.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster742452460306040201/master-1/wal
14:05:42.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster742452460306040201/master-1/data
14:05:42.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:05:42.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:05:42.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:05:42.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:05:42.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:05:42.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:05:42.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster742452460306040201/master-1/logs
14:05:42.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster742452460306040201/master-1/data/info.pb
14:05:42.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:05:42.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:05:42.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:05:42.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:05:42.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:05:42.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:05:42.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:05:42.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:05:42.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:05:42.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:05:42.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:05:42.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:05:42.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.246.253:41237
14:05:42.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.246.253
14:05:42.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:05:42.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.246.212:41037
14:05:42.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:05:42.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:05:42.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:05:42.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:05:42.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.246.254:38851,127.0.246.253:41237,127.0.246.252:36319
14:05:42.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:05:42.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44417/jwks.json with env {}
14:05:42.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:42.249330 1067 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.246.254" port: 38851 } has no permanent_uuid. Determining permanent_uuid...
14:05:42.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:42.278458 1067 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.246.253" port: 41237 } has no permanent_uuid. Determining permanent_uuid...
14:05:42.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:05:42.281170 1019 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.246.253:41237: connect: Connection refused (error 111)
14:05:42.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:05:42.285251 1067 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.0.246.253:41237: Network error: Client connection negotiation failed: client connection to 127.0.246.253:41237: connect: Connection refused (error 111)
14:05:42.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:42.334772 1067 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.246.253" port: 41237 } attempt: 1
14:05:42.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:05:42.339020 1067 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.0.246.253:41237: Network error: Client connection negotiation failed: client connection to 127.0.246.253:41237: connect: Connection refused (error 111)
14:05:42.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:42.407392 1067 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.246.253" port: 41237 } attempt: 2
14:05:42.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:05:42.411178 1067 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.0.246.253:41237: Network error: Client connection negotiation failed: client connection to 127.0.246.253:41237: connect: Connection refused (error 111)
14:05:42.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:42.502722 1067 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.246.253" port: 41237 } attempt: 3
14:05:42.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:05:42.506909 1067 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.0.246.253:41237: Network error: Client connection negotiation failed: client connection to 127.0.246.253:41237: connect: Connection refused (error 111)
14:05:42.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:05:42.561784 1069 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:05:42.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:05:42.562345 1069 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:05:42.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:05:42.562888 1069 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:05:42.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:05:42.594501 1069 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:05:42.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:05:42.594810 1069 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:05:42.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:05:42.595005 1069 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:05:42.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:05:42.595238 1069 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:05:42.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:05:42.595487 1069 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:05:42.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:05:42.595660 1069 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44417/jwks.json
14:05:42.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:42.631989 1069 master_runner.cc:386] Master server non-default flags:
14:05:42.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:05:42.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.246.212:41037
14:05:42.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:05:42.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:05:42.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster742452460306040201/master-1/data
14:05:42.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster742452460306040201/master-1/wal
14:05:42.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:05:42.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.246.254:38851,127.0.246.253:41237,127.0.246.252:36319
14:05:42.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:05:42.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:05:42.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:05:42.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.246.253:41237
14:05:42.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:05:42.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:05:42.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:05:42.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44417/jwks.json
14:05:42.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:05:42.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:05:42.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster742452460306040201/master-1/data/info.pb
14:05:42.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.246.253
14:05:42.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:05:42.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:05:42.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:05:42.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:05:42.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:05:42.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:05:42.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster742452460306040201/master-1/logs
14:05:42.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:05:42.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:05:42.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:05:42.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:05:42.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:05:42.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 06466b25e294b4d7d455e6c56263318dc34a80ef
14:05:42.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:05:42.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Sep 2025 13:43:21 UTC on 5fd53c4cbb9d
14:05:42.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7865
14:05:42.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:05:42.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:42.633388 1069 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:05:42.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:42.635478 1069 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:05:42.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:05:42.646703 1078 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:05:42.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:42.650408 1067 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.246.253" port: 41237 } attempt: 4
14:05:42.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:05:42.657539 1067 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.0.246.253:41237: Network error: Client connection negotiation failed: client connection to 127.0.246.253:41237: connect: Connection refused (error 111)
14:05:42.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:42.957087 1067 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.246.253" port: 41237 } attempt: 5
14:05:42.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:05:42.965768 1067 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.0.246.253:41237: Network error: Client connection negotiation failed: client connection to 127.0.246.253:41237: connect: Connection refused (error 111)
14:05:43.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:43.513396 1067 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.246.253" port: 41237 } attempt: 6
14:05:43.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:05:43.539361 1067 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.0.246.253:41237: Network error: Client connection negotiation failed: client connection to 127.0.246.253:41237: connect: Connection refused (error 111)
14:05:43.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:05:42.648588 1079 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:05:43.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:05:43.928714 1080 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1279 milliseconds
14:05:43.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:05:43.929314 1081 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:05:43.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:05:43.932070 1069 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.285s user 0.405s sys 0.867s
14:05:43.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:05:43.932691 1069 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.286s user 0.406s sys 0.868s
14:05:43.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:43.933008 1069 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:05:43.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:43.936228 1069 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:05:43.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:43.939267 1069 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:05:43.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:43.940688 1069 hybrid_clock.cc:648] HybridClock initialized: now 1756821943940615 us; error 89 us; skew 500 ppm
14:05:43.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:43.941922 1069 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:05:43.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:43.951306 1069 webserver.cc:480] Webserver started at http://127.0.246.253:34113/ using document root <none> and password file <none>
14:05:43.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:43.952761 1069 fs_manager.cc:362] Metadata directory not provided
14:05:43.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:43.953085 1069 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:05:43.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:43.953712 1069 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
14:05:43.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:43.960770 1069 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster742452460306040201/master-1/data/instance:
14:05:43.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "701fd5a415314ed8b9edb483649dca82"
14:05:43.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 14:05:43 on dist-test-slave-m0vh"
14:05:43.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:43.962118 1069 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster742452460306040201/master-1/wal/instance:
14:05:43.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "701fd5a415314ed8b9edb483649dca82"
14:05:43.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 14:05:43 on dist-test-slave-m0vh"
14:05:43.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:43.970016 1069 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.005s sys 0.005s
14:05:43.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:43.976308 1090 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:05:43.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:43.977532 1069 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
14:05:43.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:43.977900 1069 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster742452460306040201/master-1/data,/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster742452460306040201/master-1/wal
14:05:43.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "701fd5a415314ed8b9edb483649dca82"
14:05:43.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 14:05:43 on dist-test-slave-m0vh"
14:05:43.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:43.978273 1069 fs_report.cc:389] FS layout report
14:05:43.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:05:43.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster742452460306040201/master-1/wal
14:05:43.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster742452460306040201/master-1/wal
14:05:43.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster742452460306040201/master-1/data/data
14:05:43.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:05:43.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:05:43.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:05:43.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:05:43.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:05:43.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:05:43.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:05:43.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:05:43.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:05:43.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:05:43.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:05:44.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:44.053270 1069 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:05:44.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:44.054785 1069 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:05:44.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:44.055208 1069 kserver.cc:163] Server-wide thread pool size limit: 3276
14:05:44.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:44.126854 1069 rpc_server.cc:307] RPC server started. Bound to: 127.0.246.253:41237
14:05:44.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:44.126935 1141 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.0.246.253:41237 every 8 connection(s)
14:05:44.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:44.129792 1069 server_base.cc:1179] Dumped server information to /tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster742452460306040201/master-1/data/info.pb
14:05:44.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:44.132371 987 external_mini_cluster.cc:1428] Started /tmp/dist-test-task50Kl3p/build/tsan/bin/kudu as pid 1069
14:05:44.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:44.132891 987 external_mini_cluster.cc:1442] Reading /tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster742452460306040201/master-1/wal/instance
14:05:44.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:44.135984 987 external_mini_cluster.cc:1366] Running /tmp/dist-test-task50Kl3p/build/tsan/bin/kudu
14:05:44.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task50Kl3p/build/tsan/bin/kudu
14:05:44.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster742452460306040201/master-2/wal
14:05:44.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster742452460306040201/master-2/data
14:05:44.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:05:44.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:05:44.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:05:44.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:05:44.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:05:44.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:05:44.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster742452460306040201/master-2/logs
14:05:44.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster742452460306040201/master-2/data/info.pb
14:05:44.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:05:44.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:05:44.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:05:44.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:05:44.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:05:44.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:05:44.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:05:44.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:05:44.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:05:44.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:05:44.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:05:44.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:05:44.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.246.252:36319
14:05:44.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.246.252
14:05:44.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:05:44.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.246.212:41037
14:05:44.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:05:44.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:05:44.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:05:44.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:05:44.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.246.254:38851,127.0.246.253:41237,127.0.246.252:36319
14:05:44.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:05:44.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44417/jwks.json with env {}
14:05:44.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:44.136586 1142 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
14:05:44.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:44.156065 1142 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.246.254" port: 38851 } has no permanent_uuid. Determining permanent_uuid...
14:05:44.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:44.179780 1142 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.246.253" port: 41237 } has no permanent_uuid. Determining permanent_uuid...
14:05:44.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:44.194303 1142 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.246.252" port: 36319 } has no permanent_uuid. Determining permanent_uuid...
14:05:44.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:05:44.196591 1094 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.246.252:36319: connect: Connection refused (error 111)
14:05:44.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:05:44.200687 1142 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.0.246.252:36319: Network error: Client connection negotiation failed: client connection to 127.0.246.252:36319: connect: Connection refused (error 111)
14:05:44.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:44.250178 1142 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.246.252" port: 36319 } attempt: 1
14:05:44.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:05:44.255262 1142 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.0.246.252:36319: Network error: Client connection negotiation failed: client connection to 127.0.246.252:36319: connect: Connection refused (error 111)
14:05:44.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:44.323820 1142 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.246.252" port: 36319 } attempt: 2
14:05:44.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:05:44.328025 1142 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.0.246.252:36319: Network error: Client connection negotiation failed: client connection to 127.0.246.252:36319: connect: Connection refused (error 111)
14:05:44.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:44.419446 1142 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.246.252" port: 36319 } attempt: 3
14:05:44.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:05:44.423995 1142 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.0.246.252:36319: Network error: Client connection negotiation failed: client connection to 127.0.246.252:36319: connect: Connection refused (error 111)
14:05:44.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:05:44.461524 1144 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:05:44.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:05:44.462199 1144 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:05:44.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:05:44.462881 1144 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:05:44.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:05:44.496109 1144 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:05:44.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:05:44.496467 1144 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:05:44.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:05:44.496795 1144 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:05:44.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:05:44.497124 1144 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:05:44.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:05:44.497424 1144 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:05:44.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:05:44.497711 1144 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44417/jwks.json
14:05:44.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:44.533735 1144 master_runner.cc:386] Master server non-default flags:
14:05:44.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:05:44.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.246.212:41037
14:05:44.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:05:44.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:05:44.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster742452460306040201/master-2/data
14:05:44.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster742452460306040201/master-2/wal
14:05:44.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:05:44.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.246.254:38851,127.0.246.253:41237,127.0.246.252:36319
14:05:44.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:05:44.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:05:44.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:05:44.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.246.252:36319
14:05:44.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:05:44.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:05:44.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:05:44.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44417/jwks.json
14:05:44.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:05:44.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:05:44.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster742452460306040201/master-2/data/info.pb
14:05:44.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.246.252
14:05:44.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:05:44.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:05:44.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:05:44.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:05:44.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:05:44.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:05:44.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster742452460306040201/master-2/logs
14:05:44.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:05:44.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:05:44.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:05:44.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:05:44.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:05:44.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 06466b25e294b4d7d455e6c56263318dc34a80ef
14:05:44.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:05:44.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Sep 2025 13:43:21 UTC on 5fd53c4cbb9d
14:05:44.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7865
14:05:44.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:05:44.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:44.535595 1144 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:05:44.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:44.538607 1144 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:05:44.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:05:44.550172 1154 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:05:44.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:44.567495 1142 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.246.252" port: 36319 } attempt: 4
14:05:44.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:05:44.575141 1142 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.0.246.252:36319: Network error: Client connection negotiation failed: client connection to 127.0.246.252:36319: connect: Connection refused (error 111)
14:05:44.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:44.599967 1067 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.246.253" port: 41237 } attempt: 7
14:05:44.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:44.640635 1067 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.246.252" port: 36319 } has no permanent_uuid. Determining permanent_uuid...
14:05:44.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:05:44.664125 1067 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.0.246.252:36319: Network error: Client connection negotiation failed: client connection to 127.0.246.252:36319: connect: Connection refused (error 111)
14:05:44.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:44.722848 1067 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.246.252" port: 36319 } attempt: 1
14:05:44.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:05:44.731777 1067 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.0.246.252:36319: Network error: Client connection negotiation failed: client connection to 127.0.246.252:36319: connect: Connection refused (error 111)
14:05:44.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:44.813449 1067 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.246.252" port: 36319 } attempt: 2
14:05:44.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:05:44.822080 1067 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.0.246.252:36319: Network error: Client connection negotiation failed: client connection to 127.0.246.252:36319: connect: Connection refused (error 111)
14:05:44.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:44.875094 1142 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.246.252" port: 36319 } attempt: 5
14:05:44.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:05:44.884466 1142 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.0.246.252:36319: Network error: Client connection negotiation failed: client connection to 127.0.246.252:36319: connect: Connection refused (error 111)
14:05:44.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:44.907898 1067 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.246.252" port: 36319 } attempt: 3
14:05:44.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:05:44.916580 1067 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.0.246.252:36319: Network error: Client connection negotiation failed: client connection to 127.0.246.252:36319: connect: Connection refused (error 111)
14:05:45.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:45.057557 1067 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.246.252" port: 36319 } attempt: 4
14:05:45.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:05:45.066323 1067 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.0.246.252:36319: Network error: Client connection negotiation failed: client connection to 127.0.246.252:36319: connect: Connection refused (error 111)
14:05:45.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:45.349982 1067 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.246.252" port: 36319 } attempt: 5
14:05:45.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:05:45.358902 1067 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.0.246.252:36319: Network error: Client connection negotiation failed: client connection to 127.0.246.252:36319: connect: Connection refused (error 111)
14:05:45.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:45.432091 1142 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.246.252" port: 36319 } attempt: 6
14:05:45.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:05:45.457527 1142 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.0.246.252:36319: Network error: Client connection negotiation failed: client connection to 127.0.246.252:36319: connect: Connection refused (error 111)
14:05:45.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:05:44.551980 1155 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:05:45.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:05:45.822808 1157 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:05:45.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:05:45.824828 1156 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1268 milliseconds
14:05:45.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:05:45.825435 1144 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.275s user 0.368s sys 0.889s
14:05:45.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:05:45.825706 1144 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.275s user 0.368s sys 0.889s
14:05:45.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:45.825893 1144 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:05:45.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:45.826961 1144 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:05:45.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:45.832043 1144 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:05:45.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:45.833885 1144 hybrid_clock.cc:648] HybridClock initialized: now 1756821945833847 us; error 55 us; skew 500 ppm
14:05:45.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:45.834838 1144 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:05:45.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:45.844126 1144 webserver.cc:480] Webserver started at http://127.0.246.252:42041/ using document root <none> and password file <none>
14:05:45.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:45.845255 1144 fs_manager.cc:362] Metadata directory not provided
14:05:45.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:45.845512 1144 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:05:45.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:45.846081 1144 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
14:05:45.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:45.850822 1144 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster742452460306040201/master-2/data/instance:
14:05:45.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8bbce256c2f6402a81ebb81c4ccddd81"
14:05:45.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 14:05:45 on dist-test-slave-m0vh"
14:05:45.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:45.852103 1144 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster742452460306040201/master-2/wal/instance:
14:05:45.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8bbce256c2f6402a81ebb81c4ccddd81"
14:05:45.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 14:05:45 on dist-test-slave-m0vh"
14:05:45.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:45.860647 1144 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.006s sys 0.002s
14:05:45.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:45.867414 1168 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:05:45.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:45.868831 1144 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.002s sys 0.001s
14:05:45.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:45.869221 1144 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster742452460306040201/master-2/data,/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster742452460306040201/master-2/wal
14:05:45.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8bbce256c2f6402a81ebb81c4ccddd81"
14:05:45.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 14:05:45 on dist-test-slave-m0vh"
14:05:45.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:45.869626 1144 fs_report.cc:389] FS layout report
14:05:45.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:05:45.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster742452460306040201/master-2/wal
14:05:45.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster742452460306040201/master-2/wal
14:05:45.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster742452460306040201/master-2/data/data
14:05:45.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:05:45.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:05:45.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:05:45.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:05:45.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:05:45.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:05:45.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:05:45.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:05:45.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:05:45.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:05:45.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:05:45.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:45.911510 1067 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.246.252" port: 36319 } attempt: 6
14:05:45.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:05:45.920939 1067 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.0.246.252:36319: Network error: Client connection negotiation failed: client connection to 127.0.246.252:36319: connect: Connection refused (error 111)
14:05:45.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:45.943928 1144 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:05:45.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:45.945640 1144 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:05:45.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:45.946175 1144 kserver.cc:163] Server-wide thread pool size limit: 3276
14:05:46.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:46.018783 1144 rpc_server.cc:307] RPC server started. Bound to: 127.0.246.252:36319
14:05:46.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:46.018852 1221 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.0.246.252:36319 every 8 connection(s)
14:05:46.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:46.021812 1144 server_base.cc:1179] Dumped server information to /tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster742452460306040201/master-2/data/info.pb
14:05:46.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:46.027570 1222 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
14:05:46.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:46.032083 987 external_mini_cluster.cc:1428] Started /tmp/dist-test-task50Kl3p/build/tsan/bin/kudu as pid 1144
14:05:46.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:46.032531 987 external_mini_cluster.cc:1442] Reading /tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster742452460306040201/master-2/wal/instance
14:05:46.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:46.043152 1222 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.246.254" port: 38851 } has no permanent_uuid. Determining permanent_uuid...
14:05:46.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:46.073789 1222 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.246.253" port: 41237 } has no permanent_uuid. Determining permanent_uuid...
14:05:46.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:46.085988 1222 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.246.252" port: 36319 } has no permanent_uuid. Determining permanent_uuid...
14:05:46.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:46.104964 1222 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 8bbce256c2f6402a81ebb81c4ccddd81: Bootstrap starting.
14:05:46.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:46.111485 1222 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 8bbce256c2f6402a81ebb81c4ccddd81: Neither blocks nor log segments found. Creating new log.
14:05:46.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:46.113812 1222 log.cc:826] T 00000000000000000000000000000000 P 8bbce256c2f6402a81ebb81c4ccddd81: Log is configured to *not* fsync() on all Append() calls
14:05:46.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:46.119837 1222 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 8bbce256c2f6402a81ebb81c4ccddd81: No bootstrap required, opened a new log
14:05:46.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:46.146646 1222 raft_consensus.cc:357] T 00000000000000000000000000000000 P 8bbce256c2f6402a81ebb81c4ccddd81 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0c91631a22d842b8b15ae34f9e3fb04c" member_type: VOTER last_known_addr { host: "127.0.246.254" port: 38851 } } peers { permanent_uuid: "701fd5a415314ed8b9edb483649dca82" member_type: VOTER last_known_addr { host: "127.0.246.253" port: 41237 } } peers { permanent_uuid: "8bbce256c2f6402a81ebb81c4ccddd81" member_type: VOTER last_known_addr { host: "127.0.246.252" port: 36319 } }
14:05:46.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:46.148185 1222 raft_consensus.cc:383] T 00000000000000000000000000000000 P 8bbce256c2f6402a81ebb81c4ccddd81 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:05:46.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:46.148762 1222 raft_consensus.cc:738] T 00000000000000000000000000000000 P 8bbce256c2f6402a81ebb81c4ccddd81 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8bbce256c2f6402a81ebb81c4ccddd81, State: Initialized, Role: FOLLOWER
14:05:46.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:46.149725 1222 consensus_queue.cc:260] T 00000000000000000000000000000000 P 8bbce256c2f6402a81ebb81c4ccddd81 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0c91631a22d842b8b15ae34f9e3fb04c" member_type: VOTER last_known_addr { host: "127.0.246.254" port: 38851 } } peers { permanent_uuid: "701fd5a415314ed8b9edb483649dca82" member_type: VOTER last_known_addr { host: "127.0.246.253" port: 41237 } } peers { permanent_uuid: "8bbce256c2f6402a81ebb81c4ccddd81" member_type: VOTER last_known_addr { host: "127.0.246.252" port: 36319 } }
14:05:46.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:46.152714 1233 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8bbce256c2f6402a81ebb81c4ccddd81 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0c91631a22d842b8b15ae34f9e3fb04c" member_type: VOTER last_known_addr { host: "127.0.246.254" port: 38851 } } peers { permanent_uuid: "701fd5a415314ed8b9edb483649dca82" member_type: VOTER last_known_addr { host: "127.0.246.253" port: 41237 } } peers { permanent_uuid: "8bbce256c2f6402a81ebb81c4ccddd81" member_type: VOTER last_known_addr { host: "127.0.246.252" port: 36319 } } }
14:05:46.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:46.153333 1233 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8bbce256c2f6402a81ebb81c4ccddd81 [sys.catalog]: This master's current role is: FOLLOWER
14:05:46.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:46.154843 1222 sys_catalog.cc:564] T 00000000000000000000000000000000 P 8bbce256c2f6402a81ebb81c4ccddd81 [sys.catalog]: configured and running, proceeding with master startup.
14:05:46.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:46.180984 1233 raft_consensus.cc:491] T 00000000000000000000000000000000 P 8bbce256c2f6402a81ebb81c4ccddd81 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:05:46.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:46.181596 1233 raft_consensus.cc:513] T 00000000000000000000000000000000 P 8bbce256c2f6402a81ebb81c4ccddd81 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0c91631a22d842b8b15ae34f9e3fb04c" member_type: VOTER last_known_addr { host: "127.0.246.254" port: 38851 } } peers { permanent_uuid: "701fd5a415314ed8b9edb483649dca82" member_type: VOTER last_known_addr { host: "127.0.246.253" port: 41237 } } peers { permanent_uuid: "8bbce256c2f6402a81ebb81c4ccddd81" member_type: VOTER last_known_addr { host: "127.0.246.252" port: 36319 } }
14:05:46.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:05:46.182837 1244 catalog_manager.cc:1560] T 00000000000000000000000000000000 P 8bbce256c2f6402a81ebb81c4ccddd81: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:05:46.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:05:46.183132 1244 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:05:46.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:46.185065 1233 leader_election.cc:290] T 00000000000000000000000000000000 P 8bbce256c2f6402a81ebb81c4ccddd81 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 0c91631a22d842b8b15ae34f9e3fb04c (127.0.246.254:38851), 701fd5a415314ed8b9edb483649dca82 (127.0.246.253:41237)
14:05:46.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:05:46.202000 1176 tablet.cc:2378] T 00000000000000000000000000000000 P 8bbce256c2f6402a81ebb81c4ccddd81: Can't schedule compaction. Clean time has not been advanced past its initial value.
14:05:46.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:46.177242 1144 master_runner.cc:186] Error getting master registration for 127.0.246.254:38851: OK, instance_id { permanent_uuid: "0c91631a22d842b8b15ae34f9e3fb04c" instance_seqno: 1756821942202831 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
14:05:46.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:46.210660 1144 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:05:46.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:46.518136 1142 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.246.252" port: 36319 } attempt: 7
14:05:46.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:46.541595 1142 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 701fd5a415314ed8b9edb483649dca82: Bootstrap starting.
14:05:46.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:46.549005 1142 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 701fd5a415314ed8b9edb483649dca82: Neither blocks nor log segments found. Creating new log.
14:05:46.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:46.550905 1142 log.cc:826] T 00000000000000000000000000000000 P 701fd5a415314ed8b9edb483649dca82: Log is configured to *not* fsync() on all Append() calls
14:05:46.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:46.556667 1142 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 701fd5a415314ed8b9edb483649dca82: No bootstrap required, opened a new log
14:05:46.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:46.584408 1142 raft_consensus.cc:357] T 00000000000000000000000000000000 P 701fd5a415314ed8b9edb483649dca82 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0c91631a22d842b8b15ae34f9e3fb04c" member_type: VOTER last_known_addr { host: "127.0.246.254" port: 38851 } } peers { permanent_uuid: "701fd5a415314ed8b9edb483649dca82" member_type: VOTER last_known_addr { host: "127.0.246.253" port: 41237 } } peers { permanent_uuid: "8bbce256c2f6402a81ebb81c4ccddd81" member_type: VOTER last_known_addr { host: "127.0.246.252" port: 36319 } }
14:05:46.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:46.585361 1142 raft_consensus.cc:383] T 00000000000000000000000000000000 P 701fd5a415314ed8b9edb483649dca82 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:05:46.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:46.585668 1142 raft_consensus.cc:738] T 00000000000000000000000000000000 P 701fd5a415314ed8b9edb483649dca82 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 701fd5a415314ed8b9edb483649dca82, State: Initialized, Role: FOLLOWER
14:05:46.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:46.586572 1142 consensus_queue.cc:260] T 00000000000000000000000000000000 P 701fd5a415314ed8b9edb483649dca82 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0c91631a22d842b8b15ae34f9e3fb04c" member_type: VOTER last_known_addr { host: "127.0.246.254" port: 38851 } } peers { permanent_uuid: "701fd5a415314ed8b9edb483649dca82" member_type: VOTER last_known_addr { host: "127.0.246.253" port: 41237 } } peers { permanent_uuid: "8bbce256c2f6402a81ebb81c4ccddd81" member_type: VOTER last_known_addr { host: "127.0.246.252" port: 36319 } }
14:05:46.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:46.600924 1248 sys_catalog.cc:455] T 00000000000000000000000000000000 P 701fd5a415314ed8b9edb483649dca82 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0c91631a22d842b8b15ae34f9e3fb04c" member_type: VOTER last_known_addr { host: "127.0.246.254" port: 38851 } } peers { permanent_uuid: "701fd5a415314ed8b9edb483649dca82" member_type: VOTER last_known_addr { host: "127.0.246.253" port: 41237 } } peers { permanent_uuid: "8bbce256c2f6402a81ebb81c4ccddd81" member_type: VOTER last_known_addr { host: "127.0.246.252" port: 36319 } } }
14:05:46.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:46.601686 1248 sys_catalog.cc:458] T 00000000000000000000000000000000 P 701fd5a415314ed8b9edb483649dca82 [sys.catalog]: This master's current role is: FOLLOWER
14:05:46.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:46.602742 1142 sys_catalog.cc:564] T 00000000000000000000000000000000 P 701fd5a415314ed8b9edb483649dca82 [sys.catalog]: configured and running, proceeding with master startup.
14:05:46.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:46.602969 1117 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "8bbce256c2f6402a81ebb81c4ccddd81" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "701fd5a415314ed8b9edb483649dca82" is_pre_election: true
14:05:46.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:46.604019 1117 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 701fd5a415314ed8b9edb483649dca82 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8bbce256c2f6402a81ebb81c4ccddd81 in term 0.
14:05:46.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:46.605707 1174 leader_election.cc:304] T 00000000000000000000000000000000 P 8bbce256c2f6402a81ebb81c4ccddd81 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 701fd5a415314ed8b9edb483649dca82, 8bbce256c2f6402a81ebb81c4ccddd81; no voters:
14:05:46.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:46.606861 1233 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 8bbce256c2f6402a81ebb81c4ccddd81 [term 0 FOLLOWER]: Leader pre-election won for term 1
14:05:46.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:46.607206 1233 raft_consensus.cc:491] T 00000000000000000000000000000000 P 8bbce256c2f6402a81ebb81c4ccddd81 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
14:05:46.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:46.607766 1233 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 8bbce256c2f6402a81ebb81c4ccddd81 [term 0 FOLLOWER]: Advancing to term 1
14:05:46.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:46.614990 1233 raft_consensus.cc:513] T 00000000000000000000000000000000 P 8bbce256c2f6402a81ebb81c4ccddd81 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0c91631a22d842b8b15ae34f9e3fb04c" member_type: VOTER last_known_addr { host: "127.0.246.254" port: 38851 } } peers { permanent_uuid: "701fd5a415314ed8b9edb483649dca82" member_type: VOTER last_known_addr { host: "127.0.246.253" port: 41237 } } peers { permanent_uuid: "8bbce256c2f6402a81ebb81c4ccddd81" member_type: VOTER last_known_addr { host: "127.0.246.252" port: 36319 } }
14:05:46.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:46.628600 1117 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "8bbce256c2f6402a81ebb81c4ccddd81" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "701fd5a415314ed8b9edb483649dca82"
14:05:46.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:46.629369 1233 leader_election.cc:290] T 00000000000000000000000000000000 P 8bbce256c2f6402a81ebb81c4ccddd81 [CANDIDATE]: Term 1 election: Requested vote from peers 0c91631a22d842b8b15ae34f9e3fb04c (127.0.246.254:38851), 701fd5a415314ed8b9edb483649dca82 (127.0.246.253:41237)
14:05:46.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:46.631074 1117 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 701fd5a415314ed8b9edb483649dca82 [term 0 FOLLOWER]: Advancing to term 1
14:05:46.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:46.646268 1117 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 701fd5a415314ed8b9edb483649dca82 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8bbce256c2f6402a81ebb81c4ccddd81 in term 1.
14:05:46.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:46.647544 1174 leader_election.cc:304] T 00000000000000000000000000000000 P 8bbce256c2f6402a81ebb81c4ccddd81 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 701fd5a415314ed8b9edb483649dca82, 8bbce256c2f6402a81ebb81c4ccddd81; no voters:
14:05:46.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:46.648401 1233 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 8bbce256c2f6402a81ebb81c4ccddd81 [term 1 FOLLOWER]: Leader election won for term 1
14:05:46.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:46.665166 1233 raft_consensus.cc:695] T 00000000000000000000000000000000 P 8bbce256c2f6402a81ebb81c4ccddd81 [term 1 LEADER]: Becoming Leader. State: Replica: 8bbce256c2f6402a81ebb81c4ccddd81, State: Running, Role: LEADER
14:05:46.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:46.666146 1233 consensus_queue.cc:237] T 00000000000000000000000000000000 P 8bbce256c2f6402a81ebb81c4ccddd81 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0c91631a22d842b8b15ae34f9e3fb04c" member_type: VOTER last_known_addr { host: "127.0.246.254" port: 38851 } } peers { permanent_uuid: "701fd5a415314ed8b9edb483649dca82" member_type: VOTER last_known_addr { host: "127.0.246.253" port: 41237 } } peers { permanent_uuid: "8bbce256c2f6402a81ebb81c4ccddd81" member_type: VOTER last_known_addr { host: "127.0.246.252" port: 36319 } }
14:05:46.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:46.679212 1259 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8bbce256c2f6402a81ebb81c4ccddd81 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 8bbce256c2f6402a81ebb81c4ccddd81. Latest consensus state: current_term: 1 leader_uuid: "8bbce256c2f6402a81ebb81c4ccddd81" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0c91631a22d842b8b15ae34f9e3fb04c" member_type: VOTER last_known_addr { host: "127.0.246.254" port: 38851 } } peers { permanent_uuid: "701fd5a415314ed8b9edb483649dca82" member_type: VOTER last_known_addr { host: "127.0.246.253" port: 41237 } } peers { permanent_uuid: "8bbce256c2f6402a81ebb81c4ccddd81" member_type: VOTER last_known_addr { host: "127.0.246.252" port: 36319 } } }
14:05:46.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:46.680116 1259 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8bbce256c2f6402a81ebb81c4ccddd81 [sys.catalog]: This master's current role is: LEADER
14:05:46.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:46.687855 1263 catalog_manager.cc:1477] Loading table and tablet metadata into memory...
14:05:46.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:05:46.695273 1260 catalog_manager.cc:1560] T 00000000000000000000000000000000 P 701fd5a415314ed8b9edb483649dca82: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:05:46.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:05:46.695674 1260 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:05:46.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:46.696221 1263 catalog_manager.cc:1486] Initializing Kudu cluster ID...
14:05:46.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:46.658650 1069 master_runner.cc:186] Error getting master registration for 127.0.246.254:38851: OK, instance_id { permanent_uuid: "0c91631a22d842b8b15ae34f9e3fb04c" instance_seqno: 1756821942202831 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
14:05:46.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:46.702580 1069 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:05:46.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:46.712836 1117 raft_consensus.cc:1273] T 00000000000000000000000000000000 P 701fd5a415314ed8b9edb483649dca82 [term 1 FOLLOWER]: Refusing update from remote peer 8bbce256c2f6402a81ebb81c4ccddd81: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
14:05:46.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:46.715166 1259 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 8bbce256c2f6402a81ebb81c4ccddd81 [LEADER]: Connected to new peer: Peer: permanent_uuid: "701fd5a415314ed8b9edb483649dca82" member_type: VOTER last_known_addr { host: "127.0.246.253" port: 41237 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
14:05:46.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:46.746302 1264 mvcc.cc:204] Tried to move back new op lower bound from 7195942693717336064 to 7195942693555036160. Current Snapshot: MvccSnapshot[applied={T|T < 7195942693717336064}]
14:05:46.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:46.743496 1248 sys_catalog.cc:455] T 00000000000000000000000000000000 P 701fd5a415314ed8b9edb483649dca82 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 8bbce256c2f6402a81ebb81c4ccddd81. Latest consensus state: current_term: 1 leader_uuid: "8bbce256c2f6402a81ebb81c4ccddd81" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0c91631a22d842b8b15ae34f9e3fb04c" member_type: VOTER last_known_addr { host: "127.0.246.254" port: 38851 } } peers { permanent_uuid: "701fd5a415314ed8b9edb483649dca82" member_type: VOTER last_known_addr { host: "127.0.246.253" port: 41237 } } peers { permanent_uuid: "8bbce256c2f6402a81ebb81c4ccddd81" member_type: VOTER last_known_addr { host: "127.0.246.252" port: 36319 } } }
14:05:46.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:46.753207 1248 sys_catalog.cc:458] T 00000000000000000000000000000000 P 701fd5a415314ed8b9edb483649dca82 [sys.catalog]: This master's current role is: FOLLOWER
14:05:46.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:46.753027 1233 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8bbce256c2f6402a81ebb81c4ccddd81 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "8bbce256c2f6402a81ebb81c4ccddd81" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0c91631a22d842b8b15ae34f9e3fb04c" member_type: VOTER last_known_addr { host: "127.0.246.254" port: 38851 } } peers { permanent_uuid: "701fd5a415314ed8b9edb483649dca82" member_type: VOTER last_known_addr { host: "127.0.246.253" port: 41237 } } peers { permanent_uuid: "8bbce256c2f6402a81ebb81c4ccddd81" member_type: VOTER last_known_addr { host: "127.0.246.252" port: 36319 } } }
14:05:46.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:46.753842 1233 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8bbce256c2f6402a81ebb81c4ccddd81 [sys.catalog]: This master's current role is: LEADER
14:05:46.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:46.764441 1263 catalog_manager.cc:1349] Generated new cluster ID: 04f0c252578244198d6f9bc9983b69c9
14:05:46.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:46.764787 1263 catalog_manager.cc:1497] Initializing Kudu internal certificate authority...
14:05:46.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:46.787719 1248 sys_catalog.cc:455] T 00000000000000000000000000000000 P 701fd5a415314ed8b9edb483649dca82 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "8bbce256c2f6402a81ebb81c4ccddd81" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0c91631a22d842b8b15ae34f9e3fb04c" member_type: VOTER last_known_addr { host: "127.0.246.254" port: 38851 } } peers { permanent_uuid: "701fd5a415314ed8b9edb483649dca82" member_type: VOTER last_known_addr { host: "127.0.246.253" port: 41237 } } peers { permanent_uuid: "8bbce256c2f6402a81ebb81c4ccddd81" member_type: VOTER last_known_addr { host: "127.0.246.252" port: 36319 } } }
14:05:46.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:46.788498 1248 sys_catalog.cc:458] T 00000000000000000000000000000000 P 701fd5a415314ed8b9edb483649dca82 [sys.catalog]: This master's current role is: FOLLOWER
14:05:46.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:46.797729 1263 catalog_manager.cc:1372] Generated new certificate authority record
14:05:46.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:46.800230 1263 catalog_manager.cc:1506] Loading token signing keys...
14:05:46.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:46.831580 1263 catalog_manager.cc:5955] T 00000000000000000000000000000000 P 8bbce256c2f6402a81ebb81c4ccddd81: Generated new TSK 0
14:05:46.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:46.833359 1263 catalog_manager.cc:1516] Initializing in-progress tserver states...
14:05:46.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:46.873492 987 external_mini_cluster.cc:1366] Running /tmp/dist-test-task50Kl3p/build/tsan/bin/kudu
14:05:46.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task50Kl3p/build/tsan/bin/kudu
14:05:46.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster742452460306040201/ts-0/wal
14:05:46.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster742452460306040201/ts-0/data
14:05:46.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:05:46.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:05:46.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:05:46.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:05:46.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:05:46.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:05:46.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster742452460306040201/ts-0/logs
14:05:46.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster742452460306040201/ts-0/data/info.pb
14:05:46.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:05:46.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:05:46.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:05:46.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:05:46.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:05:46.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:05:46.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:05:46.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:05:46.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
14:05:46.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:05:46.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.246.193:0
14:05:46.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.246.193
14:05:46.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.246.193
14:05:46.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:05:46.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.246.254:38851,127.0.246.253:41237,127.0.246.252:36319
14:05:46.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.246.212:41037
14:05:46.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:05:46.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:05:46.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
14:05:46.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:46.954398 1067 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.246.252" port: 36319 } attempt: 7
14:05:47.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:47.007838 1067 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 0c91631a22d842b8b15ae34f9e3fb04c: Bootstrap starting.
14:05:47.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:47.030304 1067 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 0c91631a22d842b8b15ae34f9e3fb04c: Neither blocks nor log segments found. Creating new log.
14:05:47.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:47.032848 1067 log.cc:826] T 00000000000000000000000000000000 P 0c91631a22d842b8b15ae34f9e3fb04c: Log is configured to *not* fsync() on all Append() calls
14:05:47.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:47.047047 1067 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 0c91631a22d842b8b15ae34f9e3fb04c: No bootstrap required, opened a new log
14:05:47.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:47.078433 1067 raft_consensus.cc:357] T 00000000000000000000000000000000 P 0c91631a22d842b8b15ae34f9e3fb04c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0c91631a22d842b8b15ae34f9e3fb04c" member_type: VOTER last_known_addr { host: "127.0.246.254" port: 38851 } } peers { permanent_uuid: "701fd5a415314ed8b9edb483649dca82" member_type: VOTER last_known_addr { host: "127.0.246.253" port: 41237 } } peers { permanent_uuid: "8bbce256c2f6402a81ebb81c4ccddd81" member_type: VOTER last_known_addr { host: "127.0.246.252" port: 36319 } }
14:05:47.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:47.079356 1067 raft_consensus.cc:383] T 00000000000000000000000000000000 P 0c91631a22d842b8b15ae34f9e3fb04c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:05:47.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:47.079727 1067 raft_consensus.cc:738] T 00000000000000000000000000000000 P 0c91631a22d842b8b15ae34f9e3fb04c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 0c91631a22d842b8b15ae34f9e3fb04c, State: Initialized, Role: FOLLOWER
14:05:47.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:47.080885 1067 consensus_queue.cc:260] T 00000000000000000000000000000000 P 0c91631a22d842b8b15ae34f9e3fb04c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0c91631a22d842b8b15ae34f9e3fb04c" member_type: VOTER last_known_addr { host: "127.0.246.254" port: 38851 } } peers { permanent_uuid: "701fd5a415314ed8b9edb483649dca82" member_type: VOTER last_known_addr { host: "127.0.246.253" port: 41237 } } peers { permanent_uuid: "8bbce256c2f6402a81ebb81c4ccddd81" member_type: VOTER last_known_addr { host: "127.0.246.252" port: 36319 } }
14:05:47.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:47.089375 1273 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0c91631a22d842b8b15ae34f9e3fb04c [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0c91631a22d842b8b15ae34f9e3fb04c" member_type: VOTER last_known_addr { host: "127.0.246.254" port: 38851 } } peers { permanent_uuid: "701fd5a415314ed8b9edb483649dca82" member_type: VOTER last_known_addr { host: "127.0.246.253" port: 41237 } } peers { permanent_uuid: "8bbce256c2f6402a81ebb81c4ccddd81" member_type: VOTER last_known_addr { host: "127.0.246.252" port: 36319 } } }
14:05:47.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:47.090250 1273 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0c91631a22d842b8b15ae34f9e3fb04c [sys.catalog]: This master's current role is: FOLLOWER
14:05:47.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:47.093479 1067 sys_catalog.cc:564] T 00000000000000000000000000000000 P 0c91631a22d842b8b15ae34f9e3fb04c [sys.catalog]: configured and running, proceeding with master startup.
14:05:47.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:47.093683 1041 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "8bbce256c2f6402a81ebb81c4ccddd81" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "0c91631a22d842b8b15ae34f9e3fb04c"
14:05:47.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:47.094609 1041 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 0c91631a22d842b8b15ae34f9e3fb04c [term 0 FOLLOWER]: Advancing to term 1
14:05:47.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:47.099542 1042 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "8bbce256c2f6402a81ebb81c4ccddd81" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "0c91631a22d842b8b15ae34f9e3fb04c" is_pre_election: true
14:05:47.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:47.107760 1041 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 0c91631a22d842b8b15ae34f9e3fb04c [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8bbce256c2f6402a81ebb81c4ccddd81 in term 1.
14:05:47.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:47.112773 1040 raft_consensus.cc:1273] T 00000000000000000000000000000000 P 0c91631a22d842b8b15ae34f9e3fb04c [term 1 FOLLOWER]: Refusing update from remote peer 8bbce256c2f6402a81ebb81c4ccddd81: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
14:05:47.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:47.125758 1259 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 8bbce256c2f6402a81ebb81c4ccddd81 [LEADER]: Connected to new peer: Peer: permanent_uuid: "0c91631a22d842b8b15ae34f9e3fb04c" member_type: VOTER last_known_addr { host: "127.0.246.254" port: 38851 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
14:05:47.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:05:47.163391 1021 tablet.cc:2378] T 00000000000000000000000000000000 P 0c91631a22d842b8b15ae34f9e3fb04c: Can't schedule compaction. Clean time has not been advanced past its initial value.
14:05:47.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:47.240128 1285 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0c91631a22d842b8b15ae34f9e3fb04c [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "8bbce256c2f6402a81ebb81c4ccddd81" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0c91631a22d842b8b15ae34f9e3fb04c" member_type: VOTER last_known_addr { host: "127.0.246.254" port: 38851 } } peers { permanent_uuid: "701fd5a415314ed8b9edb483649dca82" member_type: VOTER last_known_addr { host: "127.0.246.253" port: 41237 } } peers { permanent_uuid: "8bbce256c2f6402a81ebb81c4ccddd81" member_type: VOTER last_known_addr { host: "127.0.246.252" port: 36319 } } }
14:05:47.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:47.241470 1285 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0c91631a22d842b8b15ae34f9e3fb04c [sys.catalog]: This master's current role is: FOLLOWER
14:05:47.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:47.242655 1273 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0c91631a22d842b8b15ae34f9e3fb04c [sys.catalog]: SysCatalogTable state changed. Reason: New leader 8bbce256c2f6402a81ebb81c4ccddd81. Latest consensus state: current_term: 1 leader_uuid: "8bbce256c2f6402a81ebb81c4ccddd81" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0c91631a22d842b8b15ae34f9e3fb04c" member_type: VOTER last_known_addr { host: "127.0.246.254" port: 38851 } } peers { permanent_uuid: "701fd5a415314ed8b9edb483649dca82" member_type: VOTER last_known_addr { host: "127.0.246.253" port: 41237 } } peers { permanent_uuid: "8bbce256c2f6402a81ebb81c4ccddd81" member_type: VOTER last_known_addr { host: "127.0.246.252" port: 36319 } } }
14:05:47.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:47.243544 1273 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0c91631a22d842b8b15ae34f9e3fb04c [sys.catalog]: This master's current role is: FOLLOWER
14:05:47.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:47.285399 1289 catalog_manager.cc:1261] Loaded cluster ID: 04f0c252578244198d6f9bc9983b69c9
14:05:47.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:47.285717 1289 catalog_manager.cc:1554] T 00000000000000000000000000000000 P 0c91631a22d842b8b15ae34f9e3fb04c: loading cluster ID for follower catalog manager: success
14:05:47.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:47.309326 1289 catalog_manager.cc:1576] T 00000000000000000000000000000000 P 0c91631a22d842b8b15ae34f9e3fb04c: acquiring CA information for follower catalog manager: success
14:05:47.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:47.320389 1289 catalog_manager.cc:1604] T 00000000000000000000000000000000 P 0c91631a22d842b8b15ae34f9e3fb04c: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
14:05:47.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:05:47.499476 1270 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:05:47.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:05:47.500171 1270 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:05:47.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:05:47.500999 1270 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:05:47.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:05:47.561070 1270 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:05:47.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:05:47.562482 1270 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.0.246.193
14:05:47.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:47.596853 1259 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8bbce256c2f6402a81ebb81c4ccddd81 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "8bbce256c2f6402a81ebb81c4ccddd81" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0c91631a22d842b8b15ae34f9e3fb04c" member_type: VOTER last_known_addr { host: "127.0.246.254" port: 38851 } } peers { permanent_uuid: "701fd5a415314ed8b9edb483649dca82" member_type: VOTER last_known_addr { host: "127.0.246.253" port: 41237 } } peers { permanent_uuid: "8bbce256c2f6402a81ebb81c4ccddd81" member_type: VOTER last_known_addr { host: "127.0.246.252" port: 36319 } } }
14:05:47.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:47.597955 1259 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8bbce256c2f6402a81ebb81c4ccddd81 [sys.catalog]: This master's current role is: LEADER
14:05:47.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:47.627214 1270 tablet_server_runner.cc:78] Tablet server non-default flags:
14:05:47.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:05:47.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.246.212:41037
14:05:47.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:05:47.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:05:47.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster742452460306040201/ts-0/data
14:05:47.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster742452460306040201/ts-0/wal
14:05:47.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:05:47.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:05:47.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.246.193:0
14:05:47.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:05:47.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:05:47.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:05:47.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster742452460306040201/ts-0/data/info.pb
14:05:47.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.246.193
14:05:47.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:05:47.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.246.254:38851,127.0.246.253:41237,127.0.246.252:36319
14:05:47.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:05:47.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:05:47.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:05:47.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:05:47.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:05:47.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.246.193
14:05:47.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster742452460306040201/ts-0/logs
14:05:47.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:05:47.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:05:47.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:05:47.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
14:05:47.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:05:47.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 06466b25e294b4d7d455e6c56263318dc34a80ef
14:05:47.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:05:47.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Sep 2025 13:43:21 UTC on 5fd53c4cbb9d
14:05:47.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7865
14:05:47.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:05:47.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:47.629063 1270 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:05:47.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:47.631422 1270 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:05:47.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:05:47.667435 1299 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:05:47.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:47.701498 1260 catalog_manager.cc:1261] Loaded cluster ID: 04f0c252578244198d6f9bc9983b69c9
14:05:47.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:47.702133 1260 catalog_manager.cc:1554] T 00000000000000000000000000000000 P 701fd5a415314ed8b9edb483649dca82: loading cluster ID for follower catalog manager: success
14:05:47.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:47.709769 1260 catalog_manager.cc:1576] T 00000000000000000000000000000000 P 701fd5a415314ed8b9edb483649dca82: acquiring CA information for follower catalog manager: success
14:05:47.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:47.721649 1260 catalog_manager.cc:1604] T 00000000000000000000000000000000 P 701fd5a415314ed8b9edb483649dca82: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
14:05:50.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:05:49.055219 1297 debug-util.cc:398] Leaking SignalData structure 0x7b0800034ea0 after lost signal to thread 1270
14:05:50.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:05:50.364583 1297 kernel_stack_watchdog.cc:198] Thread 1270 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 376ms:
14:05:50.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
14:05:50.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
14:05:50.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
14:05:50.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
14:05:50.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:05:47.667964 1298 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:05:50.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:05:50.370772 1270 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 2.694s user 0.003s sys 0.000s
14:05:50.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:05:50.371464 1270 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 2.695s user 0.003s sys 0.000s
14:05:50.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:05:50.372838 1301 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:05:50.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:50.374466 1270 server_base.cc:1047] running on GCE node
14:05:50.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:50.375881 1270 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:05:50.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:50.379819 1270 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:05:50.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:50.381289 1270 hybrid_clock.cc:648] HybridClock initialized: now 1756821950381157 us; error 134 us; skew 500 ppm
14:05:50.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:50.382378 1270 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:05:50.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:50.390544 1270 webserver.cc:480] Webserver started at http://127.0.246.193:37945/ using document root <none> and password file <none>
14:05:50.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:50.391911 1270 fs_manager.cc:362] Metadata directory not provided
14:05:50.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:50.392192 1270 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:05:50.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:50.392742 1270 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
14:05:50.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:50.399421 1270 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster742452460306040201/ts-0/data/instance:
14:05:50.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "41014b9fd6e2482dbf4429d5fb15973a"
14:05:50.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 14:05:50 on dist-test-slave-m0vh"
14:05:50.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:50.400935 1270 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster742452460306040201/ts-0/wal/instance:
14:05:50.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "41014b9fd6e2482dbf4429d5fb15973a"
14:05:50.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 14:05:50 on dist-test-slave-m0vh"
14:05:50.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:50.413223 1270 fs_manager.cc:696] Time spent creating directory manager: real 0.012s user 0.009s sys 0.003s
14:05:50.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:50.421043 1309 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:05:50.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:50.422370 1270 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.002s sys 0.003s
14:05:50.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:50.422775 1270 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster742452460306040201/ts-0/data,/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster742452460306040201/ts-0/wal
14:05:50.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "41014b9fd6e2482dbf4429d5fb15973a"
14:05:50.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 14:05:50 on dist-test-slave-m0vh"
14:05:50.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:50.423189 1270 fs_report.cc:389] FS layout report
14:05:50.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:05:50.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster742452460306040201/ts-0/wal
14:05:50.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster742452460306040201/ts-0/wal
14:05:50.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster742452460306040201/ts-0/data/data
14:05:50.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:05:50.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:05:50.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:05:50.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:05:50.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:05:50.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:05:50.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:05:50.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:05:50.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:05:50.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:05:50.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:05:50.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:50.484720 1270 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:05:50.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:50.486187 1270 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:05:50.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:50.486609 1270 kserver.cc:163] Server-wide thread pool size limit: 3276
14:05:50.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:50.489416 1270 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
14:05:50.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:50.494688 1270 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
14:05:50.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:50.494912 1270 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
14:05:50.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:50.495117 1270 ts_tablet_manager.cc:610] Registered 0 tablets
14:05:50.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:50.495378 1270 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.001s sys 0.000s
14:05:50.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:50.683975 1270 rpc_server.cc:307] RPC server started. Bound to: 127.0.246.193:37781
14:05:50.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:50.684098 1421 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.0.246.193:37781 every 8 connection(s)
14:05:50.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:50.686743 1270 server_base.cc:1179] Dumped server information to /tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster742452460306040201/ts-0/data/info.pb
14:05:50.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:50.694962 987 external_mini_cluster.cc:1428] Started /tmp/dist-test-task50Kl3p/build/tsan/bin/kudu as pid 1270
14:05:50.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:50.695705 987 external_mini_cluster.cc:1442] Reading /tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster742452460306040201/ts-0/wal/instance
14:05:50.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:50.729113 987 external_mini_cluster.cc:1366] Running /tmp/dist-test-task50Kl3p/build/tsan/bin/kudu
14:05:50.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task50Kl3p/build/tsan/bin/kudu
14:05:50.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster742452460306040201/ts-1/wal
14:05:50.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster742452460306040201/ts-1/data
14:05:50.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:05:50.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:05:50.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:05:50.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:05:50.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:05:50.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:05:50.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster742452460306040201/ts-1/logs
14:05:50.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster742452460306040201/ts-1/data/info.pb
14:05:50.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:05:50.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:05:50.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:05:50.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:05:50.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:05:50.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:05:50.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:05:50.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:05:50.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
14:05:50.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:05:50.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.246.194:0
14:05:50.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.246.194
14:05:50.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.246.194
14:05:50.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:05:50.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.246.254:38851,127.0.246.253:41237,127.0.246.252:36319
14:05:50.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.246.212:41037
14:05:50.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:05:50.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:05:50.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
14:05:50.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:50.753993 1422 heartbeater.cc:344] Connected to a master server at 127.0.246.252:36319
14:05:50.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:50.754544 1422 heartbeater.cc:461] Registering TS with master...
14:05:50.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:50.755961 1422 heartbeater.cc:507] Master 127.0.246.252:36319 requested a full tablet report, sending...
14:05:50.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:50.758061 1423 heartbeater.cc:344] Connected to a master server at 127.0.246.253:41237
14:05:50.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:50.758420 1423 heartbeater.cc:461] Registering TS with master...
14:05:50.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:50.759385 1423 heartbeater.cc:507] Master 127.0.246.253:41237 requested a full tablet report, sending...
14:05:50.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:50.759433 1187 ts_manager.cc:194] Registered new tserver with Master: 41014b9fd6e2482dbf4429d5fb15973a (127.0.246.193:37781)
14:05:50.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:50.762318 1107 ts_manager.cc:194] Registered new tserver with Master: 41014b9fd6e2482dbf4429d5fb15973a (127.0.246.193:37781)
14:05:50.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:50.762418 1187 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.0.246.193:55195
14:05:50.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:50.779791 1424 heartbeater.cc:344] Connected to a master server at 127.0.246.254:38851
14:05:50.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:50.780251 1424 heartbeater.cc:461] Registering TS with master...
14:05:50.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:50.780957 1424 heartbeater.cc:507] Master 127.0.246.254:38851 requested a full tablet report, sending...
14:05:50.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:50.785499 1032 ts_manager.cc:194] Registered new tserver with Master: 41014b9fd6e2482dbf4429d5fb15973a (127.0.246.193:37781)
14:05:51.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:05:51.086175 1430 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:05:51.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:05:51.086802 1430 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:05:51.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:05:51.087433 1430 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:05:51.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:05:51.123066 1430 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:05:51.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:05:51.124024 1430 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.0.246.194
14:05:51.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:51.183229 1430 tablet_server_runner.cc:78] Tablet server non-default flags:
14:05:51.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:05:51.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.246.212:41037
14:05:51.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:05:51.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:05:51.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster742452460306040201/ts-1/data
14:05:51.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster742452460306040201/ts-1/wal
14:05:51.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:05:51.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:05:51.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.246.194:0
14:05:51.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:05:51.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:05:51.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:05:51.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster742452460306040201/ts-1/data/info.pb
14:05:51.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.246.194
14:05:51.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:05:51.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.246.254:38851,127.0.246.253:41237,127.0.246.252:36319
14:05:51.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:05:51.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:05:51.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:05:51.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:05:51.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:05:51.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.246.194
14:05:51.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster742452460306040201/ts-1/logs
14:05:51.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:05:51.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:05:51.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:05:51.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
14:05:51.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:05:51.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 06466b25e294b4d7d455e6c56263318dc34a80ef
14:05:51.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:05:51.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Sep 2025 13:43:21 UTC on 5fd53c4cbb9d
14:05:51.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7865
14:05:51.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:05:51.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:51.184700 1430 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:05:51.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:51.186318 1430 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:05:51.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:05:51.200632 1439 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:05:51.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:51.767088 1422 heartbeater.cc:499] Master 127.0.246.252:36319 was elected leader, sending a full tablet report...
14:05:52.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:05:51.206794 1442 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:05:52.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:05:51.203436 1440 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:05:52.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:05:52.529898 1441 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1324 milliseconds
14:05:52.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:52.530052 1430 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:05:52.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:52.531217 1430 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:05:52.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:52.534065 1430 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:05:52.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:52.535563 1430 hybrid_clock.cc:648] HybridClock initialized: now 1756821952535500 us; error 81 us; skew 500 ppm
14:05:52.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:52.536417 1430 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:05:52.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:52.543560 1430 webserver.cc:480] Webserver started at http://127.0.246.194:35327/ using document root <none> and password file <none>
14:05:52.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:52.544502 1430 fs_manager.cc:362] Metadata directory not provided
14:05:52.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:52.544708 1430 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:05:52.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:52.545130 1430 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
14:05:52.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:52.549712 1430 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster742452460306040201/ts-1/data/instance:
14:05:52.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f522d32d634b412bb61bd89af9ee5a7b"
14:05:52.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 14:05:52 on dist-test-slave-m0vh"
14:05:52.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:52.551880 1430 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster742452460306040201/ts-1/wal/instance:
14:05:52.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f522d32d634b412bb61bd89af9ee5a7b"
14:05:52.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 14:05:52 on dist-test-slave-m0vh"
14:05:52.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:52.562299 1430 fs_manager.cc:696] Time spent creating directory manager: real 0.010s user 0.004s sys 0.004s
14:05:52.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:52.570047 1450 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:05:52.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:52.571261 1430 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.000s
14:05:52.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:52.571671 1430 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster742452460306040201/ts-1/data,/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster742452460306040201/ts-1/wal
14:05:52.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f522d32d634b412bb61bd89af9ee5a7b"
14:05:52.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 14:05:52 on dist-test-slave-m0vh"
14:05:52.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:52.571985 1430 fs_report.cc:389] FS layout report
14:05:52.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:05:52.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster742452460306040201/ts-1/wal
14:05:52.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster742452460306040201/ts-1/wal
14:05:52.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster742452460306040201/ts-1/data/data
14:05:52.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:05:52.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:05:52.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:05:52.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:05:52.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:05:52.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:05:52.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:05:52.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:05:52.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:05:52.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:05:52.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:05:52.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:52.623940 1430 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:05:52.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:52.625456 1430 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:05:52.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:52.625898 1430 kserver.cc:163] Server-wide thread pool size limit: 3276
14:05:52.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:52.628443 1430 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
14:05:52.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:52.632939 1430 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
14:05:52.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:52.633158 1430 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
14:05:52.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:52.633361 1430 ts_tablet_manager.cc:610] Registered 0 tablets
14:05:52.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:52.633496 1430 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
14:05:52.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:52.855716 1430 rpc_server.cc:307] RPC server started. Bound to: 127.0.246.194:42017
14:05:52.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:52.855909 1562 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.0.246.194:42017 every 8 connection(s)
14:05:52.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:52.858795 1430 server_base.cc:1179] Dumped server information to /tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster742452460306040201/ts-1/data/info.pb
14:05:52.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:52.859617 987 external_mini_cluster.cc:1428] Started /tmp/dist-test-task50Kl3p/build/tsan/bin/kudu as pid 1430
14:05:52.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:52.859942 987 external_mini_cluster.cc:1442] Reading /tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster742452460306040201/ts-1/wal/instance
14:05:52.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:52.872710 987 external_mini_cluster.cc:1366] Running /tmp/dist-test-task50Kl3p/build/tsan/bin/kudu
14:05:52.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task50Kl3p/build/tsan/bin/kudu
14:05:52.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster742452460306040201/ts-2/wal
14:05:52.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster742452460306040201/ts-2/data
14:05:52.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:05:52.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:05:52.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:05:52.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:05:52.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:05:52.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:05:52.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster742452460306040201/ts-2/logs
14:05:52.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster742452460306040201/ts-2/data/info.pb
14:05:52.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:05:52.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:05:52.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:05:52.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:05:52.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:05:52.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:05:52.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:05:52.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:05:52.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
14:05:52.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:05:52.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.246.195:0
14:05:52.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.246.195
14:05:52.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.246.195
14:05:52.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:05:52.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.246.254:38851,127.0.246.253:41237,127.0.246.252:36319
14:05:52.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.246.212:41037
14:05:52.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:05:52.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:05:52.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
14:05:52.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:52.928825 1564 heartbeater.cc:344] Connected to a master server at 127.0.246.253:41237
14:05:52.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:52.929443 1564 heartbeater.cc:461] Registering TS with master...
14:05:52.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:52.930910 1564 heartbeater.cc:507] Master 127.0.246.253:41237 requested a full tablet report, sending...
14:05:52.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:52.934324 1563 heartbeater.cc:344] Connected to a master server at 127.0.246.252:36319
14:05:52.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:52.934804 1563 heartbeater.cc:461] Registering TS with master...
14:05:52.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:52.935761 1567 heartbeater.cc:344] Connected to a master server at 127.0.246.254:38851
14:05:52.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:52.936218 1567 heartbeater.cc:461] Registering TS with master...
14:05:52.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:52.936389 1563 heartbeater.cc:507] Master 127.0.246.252:36319 requested a full tablet report, sending...
14:05:52.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:52.937655 1567 heartbeater.cc:507] Master 127.0.246.254:38851 requested a full tablet report, sending...
14:05:52.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:52.937942 1107 ts_manager.cc:194] Registered new tserver with Master: f522d32d634b412bb61bd89af9ee5a7b (127.0.246.194:42017)
14:05:52.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:52.939678 1187 ts_manager.cc:194] Registered new tserver with Master: f522d32d634b412bb61bd89af9ee5a7b (127.0.246.194:42017)
14:05:52.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:52.941872 1187 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.0.246.194:45835
14:05:52.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:52.942174 1032 ts_manager.cc:194] Registered new tserver with Master: f522d32d634b412bb61bd89af9ee5a7b (127.0.246.194:42017)
14:05:53.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:05:53.368827 1570 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:05:53.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:05:53.369534 1570 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:05:53.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:05:53.370002 1570 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:05:53.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:05:53.404976 1570 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:05:53.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:05:53.405820 1570 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.0.246.195
14:05:53.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:53.445327 1570 tablet_server_runner.cc:78] Tablet server non-default flags:
14:05:53.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:05:53.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.246.212:41037
14:05:53.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:05:53.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:05:53.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster742452460306040201/ts-2/data
14:05:53.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster742452460306040201/ts-2/wal
14:05:53.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:05:53.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:05:53.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.246.195:0
14:05:53.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:05:53.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:05:53.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:05:53.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster742452460306040201/ts-2/data/info.pb
14:05:53.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.246.195
14:05:53.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:05:53.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.246.254:38851,127.0.246.253:41237,127.0.246.252:36319
14:05:53.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:05:53.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:05:53.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:05:53.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:05:53.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:05:53.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.246.195
14:05:53.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster742452460306040201/ts-2/logs
14:05:53.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:05:53.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:05:53.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:05:53.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
14:05:53.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:05:53.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 06466b25e294b4d7d455e6c56263318dc34a80ef
14:05:53.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:05:53.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Sep 2025 13:43:21 UTC on 5fd53c4cbb9d
14:05:53.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7865
14:05:53.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:05:53.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:53.446815 1570 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:05:53.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:53.448614 1570 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:05:53.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:05:53.468022 1579 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:05:53.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:53.946071 1563 heartbeater.cc:499] Master 127.0.246.252:36319 was elected leader, sending a full tablet report...
14:05:54.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:05:54.505326 1138 debug-util.cc:398] Leaking SignalData structure 0x7b08000ad880 after lost signal to thread 1073
14:05:54.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:05:54.506359 1138 debug-util.cc:398] Leaking SignalData structure 0x7b0800092400 after lost signal to thread 1141
14:05:55.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:05:54.872915 1578 debug-util.cc:398] Leaking SignalData structure 0x7b0800034ea0 after lost signal to thread 1570
14:05:55.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:05:55.058881 1581 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection timed out after 1584 milliseconds
14:05:55.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:05:55.059680 1582 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:05:55.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:05:55.057055 1570 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.583s user 0.529s sys 1.031s
14:05:55.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:05:55.060243 1570 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.587s user 0.529s sys 1.031s
14:05:55.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:55.060547 1570 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:05:55.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:05:53.472839 1580 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:05:55.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:55.065480 1570 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:05:55.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:55.068514 1570 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:05:55.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:55.070005 1570 hybrid_clock.cc:648] HybridClock initialized: now 1756821955069850 us; error 160 us; skew 500 ppm
14:05:55.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:55.071177 1570 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:05:55.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:55.079453 1570 webserver.cc:480] Webserver started at http://127.0.246.195:33825/ using document root <none> and password file <none>
14:05:55.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:55.080885 1570 fs_manager.cc:362] Metadata directory not provided
14:05:55.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:55.081199 1570 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:05:55.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:55.081799 1570 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
14:05:55.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:55.089216 1570 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster742452460306040201/ts-2/data/instance:
14:05:55.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "242e4a6c10374c018a62e7241a35621c"
14:05:55.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 14:05:55 on dist-test-slave-m0vh"
14:05:55.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:55.090883 1570 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster742452460306040201/ts-2/wal/instance:
14:05:55.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "242e4a6c10374c018a62e7241a35621c"
14:05:55.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 14:05:55 on dist-test-slave-m0vh"
14:05:55.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:55.101011 1570 fs_manager.cc:696] Time spent creating directory manager: real 0.009s user 0.004s sys 0.004s
14:05:55.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:55.109341 1590 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:05:55.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:55.110560 1570 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.004s sys 0.001s
14:05:55.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:55.110879 1570 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster742452460306040201/ts-2/data,/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster742452460306040201/ts-2/wal
14:05:55.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "242e4a6c10374c018a62e7241a35621c"
14:05:55.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 14:05:55 on dist-test-slave-m0vh"
14:05:55.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:55.111189 1570 fs_report.cc:389] FS layout report
14:05:55.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:05:55.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster742452460306040201/ts-2/wal
14:05:55.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster742452460306040201/ts-2/wal
14:05:55.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster742452460306040201/ts-2/data/data
14:05:55.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:05:55.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:05:55.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:05:55.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:05:55.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:05:55.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:05:55.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:05:55.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:05:55.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:05:55.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:05:55.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:05:55.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:55.163038 1570 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:05:55.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:55.164597 1570 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:05:55.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:55.165043 1570 kserver.cc:163] Server-wide thread pool size limit: 3276
14:05:55.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:55.168061 1570 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
14:05:55.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:55.172993 1570 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
14:05:55.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:55.173210 1570 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
14:05:55.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:55.173417 1570 ts_tablet_manager.cc:610] Registered 0 tablets
14:05:55.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:55.173557 1570 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
14:05:55.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:55.780350 1570 rpc_server.cc:307] RPC server started. Bound to: 127.0.246.195:32911
14:05:55.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:55.780579 1702 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.0.246.195:32911 every 8 connection(s)
14:05:55.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:55.783900 1570 server_base.cc:1179] Dumped server information to /tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster742452460306040201/ts-2/data/info.pb
14:05:55.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:55.786729 987 external_mini_cluster.cc:1428] Started /tmp/dist-test-task50Kl3p/build/tsan/bin/kudu as pid 1570
14:05:55.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:55.787268 987 external_mini_cluster.cc:1442] Reading /tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster742452460306040201/ts-2/wal/instance
14:05:55.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:55.874656 1703 heartbeater.cc:344] Connected to a master server at 127.0.246.252:36319
14:05:55.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:55.875599 1703 heartbeater.cc:461] Registering TS with master...
14:05:55.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:55.877885 1703 heartbeater.cc:507] Master 127.0.246.252:36319 requested a full tablet report, sending...
14:05:55.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:55.883531 1187 ts_manager.cc:194] Registered new tserver with Master: 242e4a6c10374c018a62e7241a35621c (127.0.246.195:32911)
14:05:55.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:55.886622 1187 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.0.246.195:60975
14:05:55.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:55.899711 1707 heartbeater.cc:344] Connected to a master server at 127.0.246.254:38851
14:05:55.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:55.900231 1707 heartbeater.cc:461] Registering TS with master...
14:05:55.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:55.901271 1707 heartbeater.cc:507] Master 127.0.246.254:38851 requested a full tablet report, sending...
14:05:55.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:55.903656 1704 heartbeater.cc:344] Connected to a master server at 127.0.246.253:41237
14:05:55.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:55.904109 1704 heartbeater.cc:461] Registering TS with master...
14:05:55.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:55.905047 1032 ts_manager.cc:194] Registered new tserver with Master: 242e4a6c10374c018a62e7241a35621c (127.0.246.195:32911)
14:05:55.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:55.905166 1704 heartbeater.cc:507] Master 127.0.246.253:41237 requested a full tablet report, sending...
14:05:55.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:55.908885 1107 ts_manager.cc:194] Registered new tserver with Master: 242e4a6c10374c018a62e7241a35621c (127.0.246.195:32911)
14:05:55.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:55.923581 987 external_mini_cluster.cc:949] 3 TS(s) registered with all masters
14:05:55.944 [DEBUG - main] (MiniKuduCluster.java:195) Response:
14:05:55.946 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
14:05:55.959 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.0.246.254"
port: 38851
}
bound_http_address {
host: "127.0.246.254"
port: 34151
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.0.246.253"
port: 41237
}
bound_http_address {
host: "127.0.246.253"
port: 34113
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.0.246.252"
port: 36319
}
bound_http_address {
host: "127.0.246.252"
port: 42041
}
}
}
14:05:55.963 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
14:05:55.966 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.0.246.193"
port: 37781
}
bound_http_address {
host: "127.0.246.193"
port: 37945
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.0.246.194"
port: 42017
}
bound_http_address {
host: "127.0.246.194"
port: 35327
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.0.246.195"
port: 32911
}
bound_http_address {
host: "127.0.246.195"
port: 33825
}
}
}
14:05:55.966 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
14:05:56.213 [DEBUG - main] (CompositeStrategy.java:59) Unable to load modern Subject methods, relying only on legacy methods
java.lang.NoSuchMethodException: javax.security.auth.Subject.current()
at java.lang.Class.getDeclaredMethod(Class.java:2130) ~[?:1.8.0_275]
at org.apache.kudu.client.internals.ModernStrategy.<init>(ModernStrategy.java:51) ~[main/:?]
at org.apache.kudu.client.internals.CompositeStrategy.<init>(CompositeStrategy.java:52) [main/:?]
at org.apache.kudu.client.internals.CompositeStrategy.<clinit>(CompositeStrategy.java:41) [main/:?]
at org.apache.kudu.client.internals.SecurityManagerCompatibility.get(SecurityManagerCompatibility.java:43) [main/:?]
at org.apache.kudu.client.SecurityContext.setupSubject(SecurityContext.java:155) [main/:?]
at org.apache.kudu.client.SecurityContext.<init>(SecurityContext.java:138) [main/:?]
at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:431) [main/:?]
at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:299) [main/:?]
at org.apache.kudu.client.AsyncKuduClient$AsyncKuduClientBuilder.build(AsyncKuduClient.java:3109) [main/:?]
at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:161) [kudu-test-utils-1.19.0-SNAPSHOT-unshaded.jar:1.19.0-SNAPSHOT]
at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50) [junit-4.13.2.jar:4.13.2]
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184) [kudu-test-utils-1.19.0-SNAPSHOT-unshaded.jar:1.19.0-SNAPSHOT]
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244) [kudu-test-utils-1.19.0-SNAPSHOT-unshaded.jar:1.19.0-SNAPSHOT]
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184) [kudu-test-utils-1.19.0-SNAPSHOT-unshaded.jar:1.19.0-SNAPSHOT]
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244) [kudu-test-utils-1.19.0-SNAPSHOT-unshaded.jar:1.19.0-SNAPSHOT]
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.run(ParentRunner.java:413) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.Suite.runChild(Suite.java:128) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.Suite.runChild(Suite.java:27) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.run(ParentRunner.java:413) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.run(JUnitCore.java:137) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.run(JUnitCore.java:115) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.main(JUnitCore.java:36) [junit-4.13.2.jar:4.13.2]
java.lang.NoSuchMethodException: javax.security.auth.Subject.current()
at java.lang.Class.getDeclaredMethod(Class.java:2130)
at org.apache.kudu.client.internals.ModernStrategy.<init>(ModernStrategy.java:51)
at org.apache.kudu.client.internals.CompositeStrategy.<init>(CompositeStrategy.java:52)
at org.apache.kudu.client.internals.CompositeStrategy.<clinit>(CompositeStrategy.java:41)
at org.apache.kudu.client.internals.SecurityManagerCompatibility.get(SecurityManagerCompatibility.java:43)
at org.apache.kudu.client.SecurityContext.setupSubject(SecurityContext.java:155)
at org.apache.kudu.client.SecurityContext.<init>(SecurityContext.java:138)
at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:431)
at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$AsyncKuduClientBuilder.build(AsyncKuduClient.java:3109)
at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:161)
at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runners.Suite.runChild(Suite.java:128)
at org.junit.runners.Suite.runChild(Suite.java:27)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runner.JUnitCore.run(JUnitCore.java:137)
at org.junit.runner.JUnitCore.run(JUnitCore.java:115)
at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77)
at org.junit.runner.JUnitCore.main(JUnitCore.java:36)
14:05:56.250 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
14:05:56.415 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-task50Kl3p/test-tmp
14:05:56.416 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
14:05:56.416 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-task50Kl3p/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
14:05:56.419 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: true
cluster_root: "/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742"
extra_master_flags: "--webserver_require_spnego=false"
extra_master_flags: "--authn_token_validity_seconds=10"
extra_tserver_flags: "--webserver_require_spnego=false"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
14:05:56.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:56.898653 1703 heartbeater.cc:499] Master 127.0.246.252:36319 was elected leader, sending a full tablet report...
14:05:56.918 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:05:56.918 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
14:05:56.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:56.925347 1714 mini_oidc.cc:150] Starting JWKS server
14:05:56.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:56.932942 1714 webserver.cc:480] Webserver started at http://127.0.0.1:37243/ using document root <none> and password file <none>
14:05:56.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:56.933681 1714 mini_oidc.cc:186] Starting OIDC Discovery server
14:05:56.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:56.938511 1714 webserver.cc:480] Webserver started at http://127.0.0.1:36057/ using document root <none> and password file <none>
14:05:57.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
14:05:57.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/krb5kdc/principal' for realm 'KRBTEST.COM',
14:05:57.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
14:05:57.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 14:05:57 dist-test-slave-m0vh krb5kdc[1726](info): setting up network...
14:05:57.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
14:05:57.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 14:05:57 dist-test-slave-m0vh krb5kdc[1726](info): set up 2 sockets
14:05:57.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 14:05:57 dist-test-slave-m0vh krb5kdc[1726](info): commencing operation
14:05:57.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
14:05:59.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:05:59.089298 1714 mini_kdc.cc:121] Time spent starting KDC: real 2.104s user 0.003s sys 0.034s
14:05:59.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
14:05:59.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
14:05:59.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
14:05:59.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
14:05:59.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
14:05:59.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
14:05:59.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
14:05:59.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
14:05:59.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
14:05:59.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
14:05:59.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/krb5kdc/test-user.keytab.
14:05:59.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/krb5kdc/test-user.keytab.
14:05:59.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 14:05:59 dist-test-slave-m0vh krb5kdc[1726](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756821959, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:05:59.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
14:05:59.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-09-02T14:05:59Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
14:05:59.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-09-02T14:05:59Z Disabled control of system clock
14:05:59.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.1.172.190@KRBTEST.COM; defaulting to no policy
14:05:59.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:05:59.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.1.172.190@KRBTEST.COM" created.
14:05:59.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:05:59.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.172.190 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/krb5kdc/kudu.keytab.
14:05:59.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.172.190 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/krb5kdc/kudu.keytab.
14:05:59.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.1.172.190@KRBTEST.COM; defaulting to no policy
14:05:59.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:05:59.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.1.172.190@KRBTEST.COM" created.
14:05:59.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:05:59.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.1.172.190 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/krb5kdc/kudu.keytab.
14:05:59.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.1.172.190 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/krb5kdc/kudu.keytab.
14:05:59.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:59.354373 1714 external_mini_cluster.cc:1366] Running /tmp/dist-test-task50Kl3p/build/tsan/bin/kudu
14:05:59.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task50Kl3p/build/tsan/bin/kudu
14:05:59.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/master-0/wal
14:05:59.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/master-0/data
14:05:59.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:05:59.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:05:59.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:05:59.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:05:59.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:05:59.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:05:59.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/master-0/logs
14:05:59.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/master-0/data/info.pb
14:05:59.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:05:59.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:05:59.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:05:59.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:05:59.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:05:59.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:05:59.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:05:59.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:05:59.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:05:59.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:05:59.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:05:59.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:05:59.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.172.190:46329
14:05:59.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.172.190
14:05:59.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:05:59.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/krb5kdc/kudu.keytab
14:05:59.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.172.190
14:05:59.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:05:59.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:05:59.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:05:59.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
14:05:59.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.172.148:42901
14:05:59.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:05:59.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:05:59.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:05:59.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:05:59.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.172.190:46329,127.1.172.189:35607,127.1.172.188:36309
14:05:59.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:05:59.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37243/jwks.json
14:05:59.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
14:05:59.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:05:59.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:05:59.802130 1743 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:05:59.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:05:59.802747 1743 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:05:59.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:05:59.803387 1743 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:05:59.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:05:59.841380 1743 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
14:05:59.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:05:59.841655 1743 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:05:59.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:05:59.841939 1743 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:05:59.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:05:59.842145 1743 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:05:59.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:05:59.842341 1743 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:05:59.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:05:59.842535 1743 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:05:59.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:05:59.842681 1743 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37243/jwks.json
14:05:59.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:59.888842 1743 master_runner.cc:386] Master server non-default flags:
14:05:59.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:05:59.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.172.148:42901
14:05:59.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:05:59.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:05:59.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/master-0/data
14:05:59.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/master-0/wal
14:05:59.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
14:05:59.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:05:59.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.172.190:46329,127.1.172.189:35607,127.1.172.188:36309
14:05:59.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/krb5kdc/kudu.keytab
14:05:59.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.172.190
14:05:59.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:05:59.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:05:59.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:05:59.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.172.190:46329
14:05:59.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:05:59.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:05:59.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:05:59.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37243/jwks.json
14:05:59.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:05:59.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:05:59.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:05:59.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:05:59.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:05:59.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/master-0/data/info.pb
14:05:59.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.172.190
14:05:59.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:05:59.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:05:59.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:05:59.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:05:59.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:05:59.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:05:59.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/master-0/logs
14:05:59.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:05:59.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:05:59.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:05:59.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:05:59.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:05:59.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 06466b25e294b4d7d455e6c56263318dc34a80ef
14:05:59.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:05:59.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Sep 2025 13:43:21 UTC on 5fd53c4cbb9d
14:05:59.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7865
14:05:59.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:05:59.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:59.890748 1743 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:05:59.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:05:59.891855 1743 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:05:59.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:05:59.908135 1749 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:06:01.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:01.310420 1748 debug-util.cc:398] Leaking SignalData structure 0x7b0800037200 after lost signal to thread 1743
14:06:01.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:01.525024 1748 kernel_stack_watchdog.cc:198] Thread 1743 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 387ms:
14:06:01.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
14:06:01.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
14:06:01.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
14:06:01.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
14:06:01.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:01.525708 1743 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.606s user 0.557s sys 1.037s
14:06:01.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:05:59.922333 1750 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:06:01.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:01.526417 1743 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.607s user 0.557s sys 1.037s
14:06:01.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:01.528564 1752 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:06:01.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:01.530767 1751 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection timed out after 1611 milliseconds
14:06:01.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:01.530833 1743 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:06:01.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:01.532157 1743 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:06:01.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:01.534691 1743 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:06:01.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:01.536037 1743 hybrid_clock.cc:648] HybridClock initialized: now 1756821961535992 us; error 50 us; skew 500 ppm
14:06:01.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:01.536890 1743 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:06:01.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 14:06:01 dist-test-slave-m0vh krb5kdc[1726](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756821961, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.172.190@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:06:01.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:01.543038 1743 init.cc:377] Logged in from keytab as kudu/127.1.172.190@KRBTEST.COM (short username kudu)
14:06:01.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:01.547853 1743 webserver.cc:480] Webserver started at http://127.1.172.190:46345/ using document root <none> and password file <none>
14:06:01.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:01.548964 1743 fs_manager.cc:362] Metadata directory not provided
14:06:01.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:01.549198 1743 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:06:01.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:01.549732 1743 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
14:06:01.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:01.554698 1743 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/master-0/data/instance:
14:06:01.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0f05cb838d9d41ee838751c0b5839531"
14:06:01.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 14:06:01 on dist-test-slave-m0vh"
14:06:01.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:01.555994 1743 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/master-0/wal/instance:
14:06:01.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0f05cb838d9d41ee838751c0b5839531"
14:06:01.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 14:06:01 on dist-test-slave-m0vh"
14:06:01.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:01.564072 1743 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.005s sys 0.001s
14:06:01.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:01.570273 1763 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:06:01.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:01.571504 1743 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.001s
14:06:01.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:01.571919 1743 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/master-0/data,/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/master-0/wal
14:06:01.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0f05cb838d9d41ee838751c0b5839531"
14:06:01.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 14:06:01 on dist-test-slave-m0vh"
14:06:01.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:01.572278 1743 fs_report.cc:389] FS layout report
14:06:01.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:06:01.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/master-0/wal
14:06:01.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/master-0/wal
14:06:01.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/master-0/data/data
14:06:01.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:06:01.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:06:01.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:06:01.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:06:01.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:06:01.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:06:01.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:06:01.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:06:01.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:06:01.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:06:01.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:06:01.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:01.651561 1743 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:06:01.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:01.661834 1743 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:06:01.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:01.662369 1743 kserver.cc:163] Server-wide thread pool size limit: 3276
14:06:01.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:01.839888 1743 rpc_server.cc:307] RPC server started. Bound to: 127.1.172.190:46329
14:06:01.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:01.840224 1814 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.1.172.190:46329 every 8 connection(s)
14:06:01.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:01.843253 1743 server_base.cc:1179] Dumped server information to /tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/master-0/data/info.pb
14:06:01.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:01.852206 1714 external_mini_cluster.cc:1428] Started /tmp/dist-test-task50Kl3p/build/tsan/bin/kudu as pid 1743
14:06:01.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:01.852974 1714 external_mini_cluster.cc:1442] Reading /tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/master-0/wal/instance
14:06:01.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:01.858431 1815 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
14:06:01.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.1.172.189@KRBTEST.COM; defaulting to no policy
14:06:01.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:06:01.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.1.172.189@KRBTEST.COM" created.
14:06:01.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:01.880111 1815 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.172.190" port: 46329 } has no permanent_uuid. Determining permanent_uuid...
14:06:01.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:06:01.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.172.189 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/krb5kdc/kudu.keytab.
14:06:01.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.172.189 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/krb5kdc/kudu.keytab.
14:06:01.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 14:06:01 dist-test-slave-m0vh krb5kdc[1726](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756821961, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.172.190@KRBTEST.COM for kudu/127.1.172.190@KRBTEST.COM
14:06:01.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.1.172.189@KRBTEST.COM; defaulting to no policy
14:06:01.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:01.937625 1815 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.172.189" port: 35607 } has no permanent_uuid. Determining permanent_uuid...
14:06:01.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:06:01.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.1.172.189@KRBTEST.COM" created.
14:06:01.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:01.941339 1765 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.172.189:35607: connect: Connection refused (error 111)
14:06:01.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:01.948815 1815 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.1.172.189:35607: Network error: Client connection negotiation failed: client connection to 127.1.172.189:35607: connect: Connection refused (error 111)
14:06:01.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:06:01.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.1.172.189 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/krb5kdc/kudu.keytab.
14:06:01.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.1.172.189 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/krb5kdc/kudu.keytab.
14:06:01.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:01.969477 1714 external_mini_cluster.cc:1366] Running /tmp/dist-test-task50Kl3p/build/tsan/bin/kudu
14:06:01.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task50Kl3p/build/tsan/bin/kudu
14:06:01.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/master-1/wal
14:06:01.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/master-1/data
14:06:01.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:06:01.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:06:01.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:06:01.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:06:01.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:06:01.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:06:01.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/master-1/logs
14:06:01.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/master-1/data/info.pb
14:06:01.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:06:01.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:06:01.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:06:01.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:06:01.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:06:01.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:06:01.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:06:01.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:06:01.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:06:01.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:06:01.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:06:01.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:06:01.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.172.189:35607
14:06:01.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.172.189
14:06:01.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:06:01.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/krb5kdc/kudu.keytab
14:06:01.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.172.189
14:06:01.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:06:01.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:06:01.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:06:01.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
14:06:01.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.172.148:42901
14:06:01.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:06:01.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:06:01.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:06:01.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:06:01.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.172.190:46329,127.1.172.189:35607,127.1.172.188:36309
14:06:01.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:06:01.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37243/jwks.json
14:06:01.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
14:06:01.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:06:01.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:01.998507 1815 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.172.189" port: 35607 } attempt: 1
14:06:02.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:02.003767 1815 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.1.172.189:35607: Network error: Client connection negotiation failed: client connection to 127.1.172.189:35607: connect: Connection refused (error 111)
14:06:02.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:02.072185 1815 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.172.189" port: 35607 } attempt: 2
14:06:02.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:02.077559 1815 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.1.172.189:35607: Network error: Client connection negotiation failed: client connection to 127.1.172.189:35607: connect: Connection refused (error 111)
14:06:02.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:02.169086 1815 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.172.189" port: 35607 } attempt: 3
14:06:02.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:02.175436 1815 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.1.172.189:35607: Network error: Client connection negotiation failed: client connection to 127.1.172.189:35607: connect: Connection refused (error 111)
14:06:02.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:02.319013 1815 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.172.189" port: 35607 } attempt: 4
14:06:02.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:02.324388 1815 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.1.172.189:35607: Network error: Client connection negotiation failed: client connection to 127.1.172.189:35607: connect: Connection refused (error 111)
14:06:02.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:02.422189 1824 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:06:02.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:02.422847 1824 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:06:02.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:02.423858 1824 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:06:02.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:02.456385 1824 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
14:06:02.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:02.456652 1824 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:06:02.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:02.456939 1824 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:06:02.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:02.457146 1824 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:06:02.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:02.457341 1824 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:06:02.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:02.457551 1824 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:06:02.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:02.457697 1824 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37243/jwks.json
14:06:02.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:02.496222 1824 master_runner.cc:386] Master server non-default flags:
14:06:02.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:06:02.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.172.148:42901
14:06:02.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:06:02.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:06:02.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/master-1/data
14:06:02.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/master-1/wal
14:06:02.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
14:06:02.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:06:02.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.172.190:46329,127.1.172.189:35607,127.1.172.188:36309
14:06:02.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/krb5kdc/kudu.keytab
14:06:02.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.172.189
14:06:02.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:06:02.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:06:02.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:06:02.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.172.189:35607
14:06:02.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:06:02.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:06:02.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:06:02.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37243/jwks.json
14:06:02.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:06:02.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:06:02.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:06:02.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:06:02.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:06:02.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/master-1/data/info.pb
14:06:02.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.172.189
14:06:02.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:06:02.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:06:02.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:06:02.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:06:02.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:06:02.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:06:02.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/master-1/logs
14:06:02.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:06:02.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:06:02.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:06:02.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:06:02.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:06:02.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 06466b25e294b4d7d455e6c56263318dc34a80ef
14:06:02.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:06:02.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Sep 2025 13:43:21 UTC on 5fd53c4cbb9d
14:06:02.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7865
14:06:02.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:06:02.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:02.497978 1824 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:06:02.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:02.499207 1824 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:06:02.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:02.514689 1830 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:06:02.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:02.623957 1815 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.172.189" port: 35607 } attempt: 5
14:06:02.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:02.635149 1815 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.1.172.189:35607: Network error: Client connection negotiation failed: client connection to 127.1.172.189:35607: connect: Connection refused (error 111)
14:06:03.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:03.182849 1815 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.172.189" port: 35607 } attempt: 6
14:06:03.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:03.216576 1815 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.1.172.189:35607: Network error: Client connection negotiation failed: client connection to 127.1.172.189:35607: connect: Connection refused (error 111)
14:06:04.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:03.917249 1829 debug-util.cc:398] Leaking SignalData structure 0x7b0800037200 after lost signal to thread 1824
14:06:04.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:04.186967 1824 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.671s user 0.554s sys 1.017s
14:06:04.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:02.516753 1831 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:06:04.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:04.187662 1824 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.672s user 0.555s sys 1.017s
14:06:04.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:04.191074 1833 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:06:04.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:04.193140 1832 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1671 milliseconds
14:06:04.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:04.193168 1824 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:06:04.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:04.194366 1824 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:06:04.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:04.197484 1824 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:06:04.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:04.198851 1824 hybrid_clock.cc:648] HybridClock initialized: now 1756821964198817 us; error 62 us; skew 500 ppm
14:06:04.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:04.199754 1824 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:06:04.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 14:06:04 dist-test-slave-m0vh krb5kdc[1726](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756821964, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.172.189@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:06:04.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:04.206554 1824 init.cc:377] Logged in from keytab as kudu/127.1.172.189@KRBTEST.COM (short username kudu)
14:06:04.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:04.214792 1824 webserver.cc:480] Webserver started at http://127.1.172.189:33923/ using document root <none> and password file <none>
14:06:04.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:04.215878 1824 fs_manager.cc:362] Metadata directory not provided
14:06:04.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:04.216138 1824 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:06:04.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:04.216620 1824 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
14:06:04.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:04.221812 1824 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/master-1/data/instance:
14:06:04.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4cb9c6adfa244ffd8ca9ad5d2c742023"
14:06:04.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 14:06:04 on dist-test-slave-m0vh"
14:06:04.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:04.223124 1824 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/master-1/wal/instance:
14:06:04.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4cb9c6adfa244ffd8ca9ad5d2c742023"
14:06:04.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 14:06:04 on dist-test-slave-m0vh"
14:06:04.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:04.234452 1824 fs_manager.cc:696] Time spent creating directory manager: real 0.011s user 0.009s sys 0.000s
14:06:04.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:04.244055 1845 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:06:04.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:04.245514 1824 fs_manager.cc:730] Time spent opening block manager: real 0.007s user 0.002s sys 0.006s
14:06:04.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:04.245874 1824 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/master-1/data,/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/master-1/wal
14:06:04.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4cb9c6adfa244ffd8ca9ad5d2c742023"
14:06:04.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 14:06:04 on dist-test-slave-m0vh"
14:06:04.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:04.246232 1824 fs_report.cc:389] FS layout report
14:06:04.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:06:04.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/master-1/wal
14:06:04.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/master-1/wal
14:06:04.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/master-1/data/data
14:06:04.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:06:04.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:06:04.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:06:04.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:06:04.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:06:04.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:06:04.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:06:04.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:06:04.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:06:04.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:06:04.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:06:04.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:04.277328 1815 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.172.189" port: 35607 } attempt: 7
14:06:04.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:04.289403 1815 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.1.172.189:35607: Network error: Client connection negotiation failed: client connection to 127.1.172.189:35607: connect: Connection refused (error 111)
14:06:04.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:04.314666 1824 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:06:04.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:04.321664 1824 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:06:04.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:04.322304 1824 kserver.cc:163] Server-wide thread pool size limit: 3276
14:06:04.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:04.494882 1824 rpc_server.cc:307] RPC server started. Bound to: 127.1.172.189:35607
14:06:04.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:04.495113 1898 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.1.172.189:35607 every 8 connection(s)
14:06:04.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:04.498003 1824 server_base.cc:1179] Dumped server information to /tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/master-1/data/info.pb
14:06:04.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:04.503120 1714 external_mini_cluster.cc:1428] Started /tmp/dist-test-task50Kl3p/build/tsan/bin/kudu as pid 1824
14:06:04.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:04.503798 1714 external_mini_cluster.cc:1442] Reading /tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/master-1/wal/instance
14:06:04.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:04.507990 1899 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
14:06:04.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.1.172.188@KRBTEST.COM; defaulting to no policy
14:06:04.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:04.525110 1899 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.172.190" port: 46329 } has no permanent_uuid. Determining permanent_uuid...
14:06:04.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:06:04.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.1.172.188@KRBTEST.COM" created.
14:06:04.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 14:06:04 dist-test-slave-m0vh krb5kdc[1726](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756821964, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.172.189@KRBTEST.COM for kudu/127.1.172.190@KRBTEST.COM
14:06:04.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:06:04.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.172.188 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/krb5kdc/kudu.keytab.
14:06:04.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.172.188 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/krb5kdc/kudu.keytab.
14:06:04.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:04.575218 1899 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.172.189" port: 35607 } has no permanent_uuid. Determining permanent_uuid...
14:06:04.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.1.172.188@KRBTEST.COM; defaulting to no policy
14:06:04.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:06:04.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.1.172.188@KRBTEST.COM" created.
14:06:04.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 14:06:04 dist-test-slave-m0vh krb5kdc[1726](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756821964, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.172.189@KRBTEST.COM for kudu/127.1.172.189@KRBTEST.COM
14:06:04.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:04.606297 1899 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.172.188" port: 36309 } has no permanent_uuid. Determining permanent_uuid...
14:06:04.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:04.609732 1848 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.172.188:36309: connect: Connection refused (error 111)
14:06:04.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:06:04.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.1.172.188 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/krb5kdc/kudu.keytab.
14:06:04.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.1.172.188 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/krb5kdc/kudu.keytab.
14:06:04.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:04.616263 1714 external_mini_cluster.cc:1366] Running /tmp/dist-test-task50Kl3p/build/tsan/bin/kudu
14:06:04.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task50Kl3p/build/tsan/bin/kudu
14:06:04.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/master-2/wal
14:06:04.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/master-2/data
14:06:04.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:06:04.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:06:04.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:06:04.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:06:04.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:06:04.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:06:04.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/master-2/logs
14:06:04.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/master-2/data/info.pb
14:06:04.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:06:04.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:06:04.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:06:04.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:06:04.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:06:04.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:06:04.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:06:04.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:06:04.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:06:04.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:06:04.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:06:04.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:06:04.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.172.188:36309
14:06:04.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.172.188
14:06:04.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:06:04.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/krb5kdc/kudu.keytab
14:06:04.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.172.188
14:06:04.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:06:04.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:06:04.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:06:04.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
14:06:04.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.172.148:42901
14:06:04.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:06:04.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:06:04.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:06:04.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:06:04.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.172.190:46329,127.1.172.189:35607,127.1.172.188:36309
14:06:04.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:06:04.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37243/jwks.json
14:06:04.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
14:06:04.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:06:04.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:04.616827 1899 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.1.172.188:36309: Network error: Client connection negotiation failed: client connection to 127.1.172.188:36309: connect: Connection refused (error 111)
14:06:04.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:04.666343 1899 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.172.188" port: 36309 } attempt: 1
14:06:04.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:04.672533 1899 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.1.172.188:36309: Network error: Client connection negotiation failed: client connection to 127.1.172.188:36309: connect: Connection refused (error 111)
14:06:04.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:04.741016 1899 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.172.188" port: 36309 } attempt: 2
14:06:04.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:04.746618 1899 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.1.172.188:36309: Network error: Client connection negotiation failed: client connection to 127.1.172.188:36309: connect: Connection refused (error 111)
14:06:04.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:04.838219 1899 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.172.188" port: 36309 } attempt: 3
14:06:04.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:04.845657 1899 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.1.172.188:36309: Network error: Client connection negotiation failed: client connection to 127.1.172.188:36309: connect: Connection refused (error 111)
14:06:04.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:04.989230 1899 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.172.188" port: 36309 } attempt: 4
14:06:04.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:04.995878 1899 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.1.172.188:36309: Network error: Client connection negotiation failed: client connection to 127.1.172.188:36309: connect: Connection refused (error 111)
14:06:05.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:05.091986 1910 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:06:05.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:05.092590 1910 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:06:05.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:05.093207 1910 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:06:05.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:05.134642 1910 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
14:06:05.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:05.134910 1910 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:06:05.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:05.135181 1910 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:06:05.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:05.135573 1910 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:06:05.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:05.135803 1910 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:06:05.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:05.135996 1910 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:06:05.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:05.136145 1910 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37243/jwks.json
14:06:05.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:05.176808 1910 master_runner.cc:386] Master server non-default flags:
14:06:05.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:06:05.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.172.148:42901
14:06:05.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:06:05.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:06:05.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/master-2/data
14:06:05.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/master-2/wal
14:06:05.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
14:06:05.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:06:05.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.172.190:46329,127.1.172.189:35607,127.1.172.188:36309
14:06:05.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/krb5kdc/kudu.keytab
14:06:05.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.172.188
14:06:05.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:06:05.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:06:05.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:06:05.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.172.188:36309
14:06:05.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:06:05.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:06:05.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:06:05.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37243/jwks.json
14:06:05.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:06:05.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:06:05.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:06:05.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:06:05.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:06:05.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/master-2/data/info.pb
14:06:05.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.172.188
14:06:05.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:06:05.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:06:05.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:06:05.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:06:05.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:06:05.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:06:05.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/master-2/logs
14:06:05.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:06:05.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:06:05.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:06:05.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:06:05.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:06:05.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 06466b25e294b4d7d455e6c56263318dc34a80ef
14:06:05.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:06:05.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Sep 2025 13:43:21 UTC on 5fd53c4cbb9d
14:06:05.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7865
14:06:05.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:06:05.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:05.178566 1910 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:06:05.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:05.179877 1910 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:06:05.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:05.196300 1916 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:06:05.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:05.295382 1899 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.172.188" port: 36309 } attempt: 5
14:06:05.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:05.304905 1899 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.1.172.188:36309: Network error: Client connection negotiation failed: client connection to 127.1.172.188:36309: connect: Connection refused (error 111)
14:06:05.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:05.852576 1899 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.172.188" port: 36309 } attempt: 6
14:06:05.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:05.892426 1899 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.1.172.188:36309: Network error: Client connection negotiation failed: client connection to 127.1.172.188:36309: connect: Connection refused (error 111)
14:06:06.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:06.379886 1815 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.172.189" port: 35607 } attempt: 8
14:06:06.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 14:06:06 dist-test-slave-m0vh krb5kdc[1726](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756821961, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.172.190@KRBTEST.COM for kudu/127.1.172.189@KRBTEST.COM
14:06:06.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:06.436501 1815 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.172.188" port: 36309 } has no permanent_uuid. Determining permanent_uuid...
14:06:06.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:06.448310 1815 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.1.172.188:36309: Network error: Client connection negotiation failed: client connection to 127.1.172.188:36309: connect: Connection refused (error 111)
14:06:06.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:06.513870 1815 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.172.188" port: 36309 } attempt: 1
14:06:06.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:06.524648 1815 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.1.172.188:36309: Network error: Client connection negotiation failed: client connection to 127.1.172.188:36309: connect: Connection refused (error 111)
14:06:06.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:06.578310 1815 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.172.188" port: 36309 } attempt: 2
14:06:06.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:06.586452 1815 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.1.172.188:36309: Network error: Client connection negotiation failed: client connection to 127.1.172.188:36309: connect: Connection refused (error 111)
14:06:06.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:06.605043 1915 debug-util.cc:398] Leaking SignalData structure 0x7b0800037200 after lost signal to thread 1910
14:06:06.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:06.654829 1915 kernel_stack_watchdog.cc:198] Thread 1910 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 398ms:
14:06:06.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
14:06:06.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
14:06:06.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
14:06:06.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
14:06:06.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:05.200760 1917 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:06:06.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:06.659587 1910 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.457s user 0.000s sys 0.004s
14:06:06.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:06.660071 1910 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.457s user 0.000s sys 0.004s
14:06:06.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:06.663070 1815 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.172.188" port: 36309 } attempt: 3
14:06:06.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:06.669157 1919 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:06:06.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:06.669315 1815 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.1.172.188:36309: Network error: Client connection negotiation failed: client connection to 127.1.172.188:36309: connect: Connection refused (error 111)
14:06:06.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:06.670246 1910 server_base.cc:1047] running on GCE node
14:06:06.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:06.671489 1910 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:06:06.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:06.675436 1910 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:06:06.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:06.676925 1910 hybrid_clock.cc:648] HybridClock initialized: now 1756821966676911 us; error 53 us; skew 500 ppm
14:06:06.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:06.677943 1910 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:06:06.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 14:06:06 dist-test-slave-m0vh krb5kdc[1726](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756821966, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.172.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:06:06.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:06.686535 1910 init.cc:377] Logged in from keytab as kudu/127.1.172.188@KRBTEST.COM (short username kudu)
14:06:06.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:06.697312 1910 webserver.cc:480] Webserver started at http://127.1.172.188:37159/ using document root <none> and password file <none>
14:06:06.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:06.698463 1910 fs_manager.cc:362] Metadata directory not provided
14:06:06.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:06.698702 1910 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:06:06.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:06.699235 1910 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
14:06:06.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:06.704437 1910 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/master-2/data/instance:
14:06:06.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0f503058e6a84256ab9186070c48408b"
14:06:06.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 14:06:06 on dist-test-slave-m0vh"
14:06:06.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:06.705704 1910 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/master-2/wal/instance:
14:06:06.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0f503058e6a84256ab9186070c48408b"
14:06:06.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 14:06:06 on dist-test-slave-m0vh"
14:06:06.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:06.716491 1910 fs_manager.cc:696] Time spent creating directory manager: real 0.010s user 0.007s sys 0.004s
14:06:06.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:06.725878 1932 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:06:06.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:06.727486 1910 fs_manager.cc:730] Time spent opening block manager: real 0.007s user 0.002s sys 0.004s
14:06:06.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:06.727909 1910 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/master-2/data,/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/master-2/wal
14:06:06.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0f503058e6a84256ab9186070c48408b"
14:06:06.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 14:06:06 on dist-test-slave-m0vh"
14:06:06.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:06.728287 1910 fs_report.cc:389] FS layout report
14:06:06.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:06:06.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/master-2/wal
14:06:06.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/master-2/wal
14:06:06.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/master-2/data/data
14:06:06.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:06:06.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:06:06.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:06:06.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:06:06.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:06:06.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:06:06.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:06:06.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:06:06.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:06:06.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:06:06.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:06:06.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:06.794039 1910 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:06:06.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:06.804702 1910 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:06:06.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:06.805327 1910 kserver.cc:163] Server-wide thread pool size limit: 3276
14:06:06.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:06.824818 1815 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.172.188" port: 36309 } attempt: 4
14:06:06.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:06.831897 1815 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.1.172.188:36309: Network error: Client connection negotiation failed: client connection to 127.1.172.188:36309: connect: Connection refused (error 111)
14:06:06.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:06.953100 1899 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.172.188" port: 36309 } attempt: 7
14:06:06.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:06.967053 1899 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.1.172.188:36309: Network error: Client connection negotiation failed: client connection to 127.1.172.188:36309: connect: Connection refused (error 111)
14:06:06.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:06.983939 1910 rpc_server.cc:307] RPC server started. Bound to: 127.1.172.188:36309
14:06:06.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:06.984123 1986 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.1.172.188:36309 every 8 connection(s)
14:06:06.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:06.987214 1910 server_base.cc:1179] Dumped server information to /tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/master-2/data/info.pb
14:06:06.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:06.997133 1714 external_mini_cluster.cc:1428] Started /tmp/dist-test-task50Kl3p/build/tsan/bin/kudu as pid 1910
14:06:06.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:06.998454 1714 external_mini_cluster.cc:1442] Reading /tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/master-2/wal/instance
14:06:07.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:07.001518 1987 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
14:06:07.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:07.038270 1987 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.172.190" port: 46329 } has no permanent_uuid. Determining permanent_uuid...
14:06:07.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 14:06:07 dist-test-slave-m0vh krb5kdc[1726](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756821959, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.1.172.190@KRBTEST.COM
14:06:07.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 14:06:07 dist-test-slave-m0vh krb5kdc[1726](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756821966, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.172.188@KRBTEST.COM for kudu/127.1.172.190@KRBTEST.COM
14:06:07.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 14:06:07 dist-test-slave-m0vh krb5kdc[1726](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756821959, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.1.172.189@KRBTEST.COM
14:06:07.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:07.100212 1987 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.172.189" port: 35607 } has no permanent_uuid. Determining permanent_uuid...
14:06:07.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 14:06:07 dist-test-slave-m0vh krb5kdc[1726](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756821966, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.172.188@KRBTEST.COM for kudu/127.1.172.189@KRBTEST.COM
14:06:07.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 14:06:07 dist-test-slave-m0vh krb5kdc[1726](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756821959, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.1.172.188@KRBTEST.COM
14:06:07.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:07.128412 1815 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.172.188" port: 36309 } attempt: 5
14:06:07.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:07.138177 1987 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.172.188" port: 36309 } has no permanent_uuid. Determining permanent_uuid...
14:06:07.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 14:06:07 dist-test-slave-m0vh krb5kdc[1726](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756821961, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.172.190@KRBTEST.COM for kudu/127.1.172.188@KRBTEST.COM
14:06:07.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 14:06:07 dist-test-slave-m0vh krb5kdc[1726](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756821966, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.172.188@KRBTEST.COM for kudu/127.1.172.188@KRBTEST.COM
14:06:07.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:07.187268 1815 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 0f05cb838d9d41ee838751c0b5839531: Bootstrap starting.
14:06:07.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:07.191511 1987 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 0f503058e6a84256ab9186070c48408b: Bootstrap starting.
14:06:07.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:07.202126 1815 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 0f05cb838d9d41ee838751c0b5839531: Neither blocks nor log segments found. Creating new log.
14:06:07.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:07.205539 1987 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 0f503058e6a84256ab9186070c48408b: Neither blocks nor log segments found. Creating new log.
14:06:07.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:07.206106 1815 log.cc:826] T 00000000000000000000000000000000 P 0f05cb838d9d41ee838751c0b5839531: Log is configured to *not* fsync() on all Append() calls
14:06:07.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:07.209101 1987 log.cc:826] T 00000000000000000000000000000000 P 0f503058e6a84256ab9186070c48408b: Log is configured to *not* fsync() on all Append() calls
14:06:07.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:07.216322 1815 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 0f05cb838d9d41ee838751c0b5839531: No bootstrap required, opened a new log
14:06:07.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:07.225317 1987 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 0f503058e6a84256ab9186070c48408b: No bootstrap required, opened a new log
14:06:07.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:07.254096 1815 raft_consensus.cc:357] T 00000000000000000000000000000000 P 0f05cb838d9d41ee838751c0b5839531 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0f05cb838d9d41ee838751c0b5839531" member_type: VOTER last_known_addr { host: "127.1.172.190" port: 46329 } } peers { permanent_uuid: "4cb9c6adfa244ffd8ca9ad5d2c742023" member_type: VOTER last_known_addr { host: "127.1.172.189" port: 35607 } } peers { permanent_uuid: "0f503058e6a84256ab9186070c48408b" member_type: VOTER last_known_addr { host: "127.1.172.188" port: 36309 } }
14:06:07.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:07.254978 1815 raft_consensus.cc:383] T 00000000000000000000000000000000 P 0f05cb838d9d41ee838751c0b5839531 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:06:07.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:07.255406 1815 raft_consensus.cc:738] T 00000000000000000000000000000000 P 0f05cb838d9d41ee838751c0b5839531 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 0f05cb838d9d41ee838751c0b5839531, State: Initialized, Role: FOLLOWER
14:06:07.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:07.256656 1815 consensus_queue.cc:260] T 00000000000000000000000000000000 P 0f05cb838d9d41ee838751c0b5839531 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0f05cb838d9d41ee838751c0b5839531" member_type: VOTER last_known_addr { host: "127.1.172.190" port: 46329 } } peers { permanent_uuid: "4cb9c6adfa244ffd8ca9ad5d2c742023" member_type: VOTER last_known_addr { host: "127.1.172.189" port: 35607 } } peers { permanent_uuid: "0f503058e6a84256ab9186070c48408b" member_type: VOTER last_known_addr { host: "127.1.172.188" port: 36309 } }
14:06:07.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:07.267721 1815 sys_catalog.cc:564] T 00000000000000000000000000000000 P 0f05cb838d9d41ee838751c0b5839531 [sys.catalog]: configured and running, proceeding with master startup.
14:06:07.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:07.268194 1987 raft_consensus.cc:357] T 00000000000000000000000000000000 P 0f503058e6a84256ab9186070c48408b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0f05cb838d9d41ee838751c0b5839531" member_type: VOTER last_known_addr { host: "127.1.172.190" port: 46329 } } peers { permanent_uuid: "4cb9c6adfa244ffd8ca9ad5d2c742023" member_type: VOTER last_known_addr { host: "127.1.172.189" port: 35607 } } peers { permanent_uuid: "0f503058e6a84256ab9186070c48408b" member_type: VOTER last_known_addr { host: "127.1.172.188" port: 36309 } }
14:06:07.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:07.269423 1987 raft_consensus.cc:383] T 00000000000000000000000000000000 P 0f503058e6a84256ab9186070c48408b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:06:07.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:07.269807 1987 raft_consensus.cc:738] T 00000000000000000000000000000000 P 0f503058e6a84256ab9186070c48408b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 0f503058e6a84256ab9186070c48408b, State: Initialized, Role: FOLLOWER
14:06:07.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:07.270959 1987 consensus_queue.cc:260] T 00000000000000000000000000000000 P 0f503058e6a84256ab9186070c48408b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0f05cb838d9d41ee838751c0b5839531" member_type: VOTER last_known_addr { host: "127.1.172.190" port: 46329 } } peers { permanent_uuid: "4cb9c6adfa244ffd8ca9ad5d2c742023" member_type: VOTER last_known_addr { host: "127.1.172.189" port: 35607 } } peers { permanent_uuid: "0f503058e6a84256ab9186070c48408b" member_type: VOTER last_known_addr { host: "127.1.172.188" port: 36309 } }
14:06:07.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:07.273608 2000 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0f05cb838d9d41ee838751c0b5839531 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0f05cb838d9d41ee838751c0b5839531" member_type: VOTER last_known_addr { host: "127.1.172.190" port: 46329 } } peers { permanent_uuid: "4cb9c6adfa244ffd8ca9ad5d2c742023" member_type: VOTER last_known_addr { host: "127.1.172.189" port: 35607 } } peers { permanent_uuid: "0f503058e6a84256ab9186070c48408b" member_type: VOTER last_known_addr { host: "127.1.172.188" port: 36309 } } }
14:06:07.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:07.275607 2000 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0f05cb838d9d41ee838751c0b5839531 [sys.catalog]: This master's current role is: FOLLOWER
14:06:07.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:07.281268 2001 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0f503058e6a84256ab9186070c48408b [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0f05cb838d9d41ee838751c0b5839531" member_type: VOTER last_known_addr { host: "127.1.172.190" port: 46329 } } peers { permanent_uuid: "4cb9c6adfa244ffd8ca9ad5d2c742023" member_type: VOTER last_known_addr { host: "127.1.172.189" port: 35607 } } peers { permanent_uuid: "0f503058e6a84256ab9186070c48408b" member_type: VOTER last_known_addr { host: "127.1.172.188" port: 36309 } } }
14:06:07.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:07.282830 2001 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0f503058e6a84256ab9186070c48408b [sys.catalog]: This master's current role is: FOLLOWER
14:06:07.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:07.285391 1987 sys_catalog.cc:564] T 00000000000000000000000000000000 P 0f503058e6a84256ab9186070c48408b [sys.catalog]: configured and running, proceeding with master startup.
14:06:07.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:07.320937 1938 tablet.cc:2378] T 00000000000000000000000000000000 P 0f503058e6a84256ab9186070c48408b: Can't schedule compaction. Clean time has not been advanced past its initial value.
14:06:07.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:07.356127 2019 catalog_manager.cc:1560] T 00000000000000000000000000000000 P 0f05cb838d9d41ee838751c0b5839531: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:06:07.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:07.362254 2019 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:06:07.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:07.346621 1743 master_runner.cc:186] Error getting master registration for 127.1.172.189:35607: OK, instance_id { permanent_uuid: "4cb9c6adfa244ffd8ca9ad5d2c742023" instance_seqno: 1756821964445884 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
14:06:07.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:07.386234 1743 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:06:07.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:07.391458 2024 catalog_manager.cc:1560] T 00000000000000000000000000000000 P 0f503058e6a84256ab9186070c48408b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:06:07.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:07.391932 2024 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:06:07.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:07.426777 1769 tablet.cc:2378] T 00000000000000000000000000000000 P 0f05cb838d9d41ee838751c0b5839531: Can't schedule compaction. Clean time has not been advanced past its initial value.
14:06:07.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:07.376025 1910 master_runner.cc:186] Error getting master registration for 127.1.172.189:35607: OK, instance_id { permanent_uuid: "4cb9c6adfa244ffd8ca9ad5d2c742023" instance_seqno: 1756821964445884 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
14:06:07.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:07.432885 1910 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:06:07.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:07.679535 2000 raft_consensus.cc:491] T 00000000000000000000000000000000 P 0f05cb838d9d41ee838751c0b5839531 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:06:07.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:07.680130 2000 raft_consensus.cc:513] T 00000000000000000000000000000000 P 0f05cb838d9d41ee838751c0b5839531 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0f05cb838d9d41ee838751c0b5839531" member_type: VOTER last_known_addr { host: "127.1.172.190" port: 46329 } } peers { permanent_uuid: "4cb9c6adfa244ffd8ca9ad5d2c742023" member_type: VOTER last_known_addr { host: "127.1.172.189" port: 35607 } } peers { permanent_uuid: "0f503058e6a84256ab9186070c48408b" member_type: VOTER last_known_addr { host: "127.1.172.188" port: 36309 } }
14:06:07.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:07.686118 1960 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "0f05cb838d9d41ee838751c0b5839531" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "0f503058e6a84256ab9186070c48408b" is_pre_election: true
14:06:07.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:07.690634 1960 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 0f503058e6a84256ab9186070c48408b [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 0f05cb838d9d41ee838751c0b5839531 in term 0.
14:06:07.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:07.693053 1764 leader_election.cc:304] T 00000000000000000000000000000000 P 0f05cb838d9d41ee838751c0b5839531 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 0f05cb838d9d41ee838751c0b5839531, 0f503058e6a84256ab9186070c48408b; no voters:
14:06:07.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:07.694504 2000 leader_election.cc:290] T 00000000000000000000000000000000 P 0f05cb838d9d41ee838751c0b5839531 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 4cb9c6adfa244ffd8ca9ad5d2c742023 (127.1.172.189:35607), 0f503058e6a84256ab9186070c48408b (127.1.172.188:36309)
14:06:07.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:07.695053 2000 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 0f05cb838d9d41ee838751c0b5839531 [term 0 FOLLOWER]: Leader pre-election won for term 1
14:06:07.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:07.695657 2000 raft_consensus.cc:491] T 00000000000000000000000000000000 P 0f05cb838d9d41ee838751c0b5839531 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
14:06:07.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:07.695997 2000 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 0f05cb838d9d41ee838751c0b5839531 [term 0 FOLLOWER]: Advancing to term 1
14:06:07.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:07.712983 2000 raft_consensus.cc:513] T 00000000000000000000000000000000 P 0f05cb838d9d41ee838751c0b5839531 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0f05cb838d9d41ee838751c0b5839531" member_type: VOTER last_known_addr { host: "127.1.172.190" port: 46329 } } peers { permanent_uuid: "4cb9c6adfa244ffd8ca9ad5d2c742023" member_type: VOTER last_known_addr { host: "127.1.172.189" port: 35607 } } peers { permanent_uuid: "0f503058e6a84256ab9186070c48408b" member_type: VOTER last_known_addr { host: "127.1.172.188" port: 36309 } }
14:06:07.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:07.721200 1960 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "0f05cb838d9d41ee838751c0b5839531" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "0f503058e6a84256ab9186070c48408b"
14:06:07.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:07.721967 1960 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 0f503058e6a84256ab9186070c48408b [term 0 FOLLOWER]: Advancing to term 1
14:06:07.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:07.723951 2000 leader_election.cc:290] T 00000000000000000000000000000000 P 0f05cb838d9d41ee838751c0b5839531 [CANDIDATE]: Term 1 election: Requested vote from peers 4cb9c6adfa244ffd8ca9ad5d2c742023 (127.1.172.189:35607), 0f503058e6a84256ab9186070c48408b (127.1.172.188:36309)
14:06:07.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:07.732796 1960 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 0f503058e6a84256ab9186070c48408b [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 0f05cb838d9d41ee838751c0b5839531 in term 1.
14:06:07.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:07.743778 1764 leader_election.cc:304] T 00000000000000000000000000000000 P 0f05cb838d9d41ee838751c0b5839531 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 0f05cb838d9d41ee838751c0b5839531, 0f503058e6a84256ab9186070c48408b; no voters:
14:06:07.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:07.744944 2000 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 0f05cb838d9d41ee838751c0b5839531 [term 1 FOLLOWER]: Leader election won for term 1
14:06:07.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:07.745551 2000 raft_consensus.cc:695] T 00000000000000000000000000000000 P 0f05cb838d9d41ee838751c0b5839531 [term 1 LEADER]: Becoming Leader. State: Replica: 0f05cb838d9d41ee838751c0b5839531, State: Running, Role: LEADER
14:06:07.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:07.746852 2000 consensus_queue.cc:237] T 00000000000000000000000000000000 P 0f05cb838d9d41ee838751c0b5839531 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0f05cb838d9d41ee838751c0b5839531" member_type: VOTER last_known_addr { host: "127.1.172.190" port: 46329 } } peers { permanent_uuid: "4cb9c6adfa244ffd8ca9ad5d2c742023" member_type: VOTER last_known_addr { host: "127.1.172.189" port: 35607 } } peers { permanent_uuid: "0f503058e6a84256ab9186070c48408b" member_type: VOTER last_known_addr { host: "127.1.172.188" port: 36309 } }
14:06:07.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:07.770277 2027 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0f05cb838d9d41ee838751c0b5839531 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 0f05cb838d9d41ee838751c0b5839531. Latest consensus state: current_term: 1 leader_uuid: "0f05cb838d9d41ee838751c0b5839531" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0f05cb838d9d41ee838751c0b5839531" member_type: VOTER last_known_addr { host: "127.1.172.190" port: 46329 } } peers { permanent_uuid: "4cb9c6adfa244ffd8ca9ad5d2c742023" member_type: VOTER last_known_addr { host: "127.1.172.189" port: 35607 } } peers { permanent_uuid: "0f503058e6a84256ab9186070c48408b" member_type: VOTER last_known_addr { host: "127.1.172.188" port: 36309 } } }
14:06:07.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:07.771688 2027 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0f05cb838d9d41ee838751c0b5839531 [sys.catalog]: This master's current role is: LEADER
14:06:07.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:07.780639 2029 catalog_manager.cc:1477] Loading table and tablet metadata into memory...
14:06:07.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:07.794200 2029 catalog_manager.cc:1486] Initializing Kudu cluster ID...
14:06:07.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:07.824458 1960 raft_consensus.cc:1273] T 00000000000000000000000000000000 P 0f503058e6a84256ab9186070c48408b [term 1 FOLLOWER]: Refusing update from remote peer 0f05cb838d9d41ee838751c0b5839531: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
14:06:07.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:07.828783 2000 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 0f05cb838d9d41ee838751c0b5839531 [LEADER]: Connected to new peer: Peer: permanent_uuid: "0f503058e6a84256ab9186070c48408b" member_type: VOTER last_known_addr { host: "127.1.172.188" port: 36309 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
14:06:07.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:07.902609 2031 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0f503058e6a84256ab9186070c48408b [sys.catalog]: SysCatalogTable state changed. Reason: New leader 0f05cb838d9d41ee838751c0b5839531. Latest consensus state: current_term: 1 leader_uuid: "0f05cb838d9d41ee838751c0b5839531" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0f05cb838d9d41ee838751c0b5839531" member_type: VOTER last_known_addr { host: "127.1.172.190" port: 46329 } } peers { permanent_uuid: "4cb9c6adfa244ffd8ca9ad5d2c742023" member_type: VOTER last_known_addr { host: "127.1.172.189" port: 35607 } } peers { permanent_uuid: "0f503058e6a84256ab9186070c48408b" member_type: VOTER last_known_addr { host: "127.1.172.188" port: 36309 } } }
14:06:07.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:07.905817 2031 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0f503058e6a84256ab9186070c48408b [sys.catalog]: This master's current role is: FOLLOWER
14:06:07.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:07.917639 2030 mvcc.cc:204] Tried to move back new op lower bound from 7195942780177801216 to 7195942779915137024. Current Snapshot: MvccSnapshot[applied={T|T < 7195942780177801216}]
14:06:07.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:07.939268 2027 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0f05cb838d9d41ee838751c0b5839531 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "0f05cb838d9d41ee838751c0b5839531" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0f05cb838d9d41ee838751c0b5839531" member_type: VOTER last_known_addr { host: "127.1.172.190" port: 46329 } } peers { permanent_uuid: "4cb9c6adfa244ffd8ca9ad5d2c742023" member_type: VOTER last_known_addr { host: "127.1.172.189" port: 35607 } } peers { permanent_uuid: "0f503058e6a84256ab9186070c48408b" member_type: VOTER last_known_addr { host: "127.1.172.188" port: 36309 } } }
14:06:07.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:07.940397 2027 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0f05cb838d9d41ee838751c0b5839531 [sys.catalog]: This master's current role is: LEADER
14:06:07.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:07.939983 2031 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0f503058e6a84256ab9186070c48408b [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "0f05cb838d9d41ee838751c0b5839531" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0f05cb838d9d41ee838751c0b5839531" member_type: VOTER last_known_addr { host: "127.1.172.190" port: 46329 } } peers { permanent_uuid: "4cb9c6adfa244ffd8ca9ad5d2c742023" member_type: VOTER last_known_addr { host: "127.1.172.189" port: 35607 } } peers { permanent_uuid: "0f503058e6a84256ab9186070c48408b" member_type: VOTER last_known_addr { host: "127.1.172.188" port: 36309 } } }
14:06:07.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:07.940866 2031 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0f503058e6a84256ab9186070c48408b [sys.catalog]: This master's current role is: FOLLOWER
14:06:07.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:07.963891 2029 catalog_manager.cc:1349] Generated new cluster ID: 0c29294b4d6c4be5afc6c01b4bfa41bd
14:06:07.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:07.964345 2029 catalog_manager.cc:1497] Initializing Kudu internal certificate authority...
14:06:08.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:08.006894 2029 catalog_manager.cc:1372] Generated new certificate authority record
14:06:08.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:08.009521 2029 catalog_manager.cc:1506] Loading token signing keys...
14:06:08.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:08.080587 2029 catalog_manager.cc:5955] T 00000000000000000000000000000000 P 0f05cb838d9d41ee838751c0b5839531: Generated new TSK 0
14:06:08.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:08.081905 2029 catalog_manager.cc:1516] Initializing in-progress tserver states...
14:06:08.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.1.172.129@KRBTEST.COM; defaulting to no policy
14:06:08.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:06:08.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.1.172.129@KRBTEST.COM" created.
14:06:08.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:06:08.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.172.129 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/krb5kdc/kudu.keytab.
14:06:08.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.172.129 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/krb5kdc/kudu.keytab.
14:06:08.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.1.172.129@KRBTEST.COM; defaulting to no policy
14:06:08.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:06:08.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.1.172.129@KRBTEST.COM" created.
14:06:08.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:06:08.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.1.172.129 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/krb5kdc/kudu.keytab.
14:06:08.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.1.172.129 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/krb5kdc/kudu.keytab.
14:06:08.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:08.326922 1714 external_mini_cluster.cc:1366] Running /tmp/dist-test-task50Kl3p/build/tsan/bin/kudu
14:06:08.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task50Kl3p/build/tsan/bin/kudu
14:06:08.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/ts-0/wal
14:06:08.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/ts-0/data
14:06:08.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:06:08.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:06:08.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:06:08.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:06:08.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:06:08.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:06:08.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/ts-0/logs
14:06:08.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/ts-0/data/info.pb
14:06:08.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:06:08.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:06:08.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:06:08.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:06:08.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:06:08.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:06:08.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:06:08.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:06:08.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
14:06:08.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:06:08.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.172.129:0
14:06:08.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.172.129
14:06:08.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.172.129
14:06:08.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:06:08.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.172.190:46329,127.1.172.189:35607,127.1.172.188:36309
14:06:08.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/krb5kdc/kudu.keytab
14:06:08.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.172.129
14:06:08.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:06:08.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:06:08.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:06:08.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
14:06:08.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.172.148:42901
14:06:08.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:06:08.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:06:08.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:06:08.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false with env {KRB5CCNAME=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:06:08.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:08.399451 2024 catalog_manager.cc:1261] Loaded cluster ID: 0c29294b4d6c4be5afc6c01b4bfa41bd
14:06:08.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:08.399861 2024 catalog_manager.cc:1554] T 00000000000000000000000000000000 P 0f503058e6a84256ab9186070c48408b: loading cluster ID for follower catalog manager: success
14:06:08.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:08.427584 2024 catalog_manager.cc:1576] T 00000000000000000000000000000000 P 0f503058e6a84256ab9186070c48408b: acquiring CA information for follower catalog manager: success
14:06:08.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:08.434870 2024 catalog_manager.cc:1604] T 00000000000000000000000000000000 P 0f503058e6a84256ab9186070c48408b: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
14:06:09.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:09.057670 1899 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.172.188" port: 36309 } attempt: 8
14:06:09.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:09.091567 2040 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:06:09.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:09.092312 2040 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:06:09.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:09.093385 2040 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:06:09.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 14:06:09 dist-test-slave-m0vh krb5kdc[1726](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756821964, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.172.189@KRBTEST.COM for kudu/127.1.172.188@KRBTEST.COM
14:06:09.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:09.154629 2040 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:06:09.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:09.156394 2040 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.1.172.129
14:06:09.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:09.157553 1899 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4cb9c6adfa244ffd8ca9ad5d2c742023: Bootstrap starting.
14:06:09.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:09.189110 1899 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 4cb9c6adfa244ffd8ca9ad5d2c742023: Neither blocks nor log segments found. Creating new log.
14:06:09.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:09.192130 1899 log.cc:826] T 00000000000000000000000000000000 P 4cb9c6adfa244ffd8ca9ad5d2c742023: Log is configured to *not* fsync() on all Append() calls
14:06:09.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:09.220851 1899 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4cb9c6adfa244ffd8ca9ad5d2c742023: No bootstrap required, opened a new log
14:06:09.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:09.227487 2040 tablet_server_runner.cc:78] Tablet server non-default flags:
14:06:09.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:06:09.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.172.148:42901
14:06:09.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:06:09.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:06:09.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/ts-0/data
14:06:09.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/ts-0/wal
14:06:09.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/krb5kdc/kudu.keytab
14:06:09.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.172.129
14:06:09.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:06:09.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:06:09.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.172.129:0
14:06:09.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:06:09.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:06:09.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:06:09.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:06:09.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:06:09.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:06:09.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/ts-0/data/info.pb
14:06:09.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.172.129
14:06:09.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:06:09.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.172.190:46329,127.1.172.189:35607,127.1.172.188:36309
14:06:09.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:06:09.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:06:09.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:06:09.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:06:09.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:06:09.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.172.129
14:06:09.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/ts-0/logs
14:06:09.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:06:09.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:06:09.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:06:09.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
14:06:09.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:06:09.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 06466b25e294b4d7d455e6c56263318dc34a80ef
14:06:09.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:06:09.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Sep 2025 13:43:21 UTC on 5fd53c4cbb9d
14:06:09.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7865
14:06:09.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:06:09.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:09.229260 2040 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:06:09.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:09.230629 2040 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:06:09.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:09.231823 1765 proxy.cc:239] Call had error, refreshing address and retrying: Timed out: RequestConsensusVote RPC to 127.1.172.189:35607 timed out after 1.546s (SENT) [suppressed 12 similar messages]
14:06:09.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:09.289112 2050 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:06:09.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:09.286713 1899 raft_consensus.cc:357] T 00000000000000000000000000000000 P 4cb9c6adfa244ffd8ca9ad5d2c742023 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0f05cb838d9d41ee838751c0b5839531" member_type: VOTER last_known_addr { host: "127.1.172.190" port: 46329 } } peers { permanent_uuid: "4cb9c6adfa244ffd8ca9ad5d2c742023" member_type: VOTER last_known_addr { host: "127.1.172.189" port: 35607 } } peers { permanent_uuid: "0f503058e6a84256ab9186070c48408b" member_type: VOTER last_known_addr { host: "127.1.172.188" port: 36309 } }
14:06:09.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:09.290678 1899 raft_consensus.cc:383] T 00000000000000000000000000000000 P 4cb9c6adfa244ffd8ca9ad5d2c742023 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:06:09.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:09.291155 1899 raft_consensus.cc:738] T 00000000000000000000000000000000 P 4cb9c6adfa244ffd8ca9ad5d2c742023 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4cb9c6adfa244ffd8ca9ad5d2c742023, State: Initialized, Role: FOLLOWER
14:06:09.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:09.292742 1899 consensus_queue.cc:260] T 00000000000000000000000000000000 P 4cb9c6adfa244ffd8ca9ad5d2c742023 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0f05cb838d9d41ee838751c0b5839531" member_type: VOTER last_known_addr { host: "127.1.172.190" port: 46329 } } peers { permanent_uuid: "4cb9c6adfa244ffd8ca9ad5d2c742023" member_type: VOTER last_known_addr { host: "127.1.172.189" port: 35607 } } peers { permanent_uuid: "0f503058e6a84256ab9186070c48408b" member_type: VOTER last_known_addr { host: "127.1.172.188" port: 36309 } }
14:06:09.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:09.313138 1899 sys_catalog.cc:564] T 00000000000000000000000000000000 P 4cb9c6adfa244ffd8ca9ad5d2c742023 [sys.catalog]: configured and running, proceeding with master startup.
14:06:09.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:09.314968 1872 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 4cb9c6adfa244ffd8ca9ad5d2c742023 [term 0 FOLLOWER]: Advancing to term 1
14:06:09.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:09.310254 2053 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4cb9c6adfa244ffd8ca9ad5d2c742023 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0f05cb838d9d41ee838751c0b5839531" member_type: VOTER last_known_addr { host: "127.1.172.190" port: 46329 } } peers { permanent_uuid: "4cb9c6adfa244ffd8ca9ad5d2c742023" member_type: VOTER last_known_addr { host: "127.1.172.189" port: 35607 } } peers { permanent_uuid: "0f503058e6a84256ab9186070c48408b" member_type: VOTER last_known_addr { host: "127.1.172.188" port: 36309 } } }
14:06:09.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:09.323803 2053 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4cb9c6adfa244ffd8ca9ad5d2c742023 [sys.catalog]: This master's current role is: FOLLOWER
14:06:09.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:09.324860 1874 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "0f05cb838d9d41ee838751c0b5839531" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4cb9c6adfa244ffd8ca9ad5d2c742023" is_pre_election: true
14:06:09.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:09.328291 1871 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "0f05cb838d9d41ee838751c0b5839531" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4cb9c6adfa244ffd8ca9ad5d2c742023" is_pre_election: true
14:06:09.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:09.333504 1874 rpcz_store.cc:267] Call kudu.consensus.ConsensusService.RequestConsensusVote from 127.0.0.1:58710 (request call id 2) took 1645 ms (client timeout 1546 ms). Trace:
14:06:09.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:09.335508 1874 rpcz_store.cc:269] 0902 14:06:07.688230 (+ 0us) service_pool.cc:168] Inserting onto call queue
14:06:09.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 14:06:07.688756 (+ 526us) service_pool.cc:225] Handling call
14:06:09.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 14:06:09.333200 (+1644444us) inbound_call.cc:173] Queueing success response
14:06:09.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
14:06:09.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:09.320842 1873 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "0f05cb838d9d41ee838751c0b5839531" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4cb9c6adfa244ffd8ca9ad5d2c742023"
14:06:09.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:09.340468 1873 rpcz_store.cc:267] Call kudu.consensus.ConsensusService.RequestConsensusVote from 127.0.0.1:58710 (request call id 3) took 1621 ms (client timeout 1757 ms). Trace:
14:06:09.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:09.340900 1853 tablet.cc:2378] T 00000000000000000000000000000000 P 4cb9c6adfa244ffd8ca9ad5d2c742023: Can't schedule compaction. Clean time has not been advanced past its initial value.
14:06:09.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:09.341073 1873 rpcz_store.cc:269] 0902 14:06:07.719114 (+ 0us) service_pool.cc:168] Inserting onto call queue
14:06:09.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 14:06:07.720260 (+ 1146us) service_pool.cc:225] Handling call
14:06:09.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 14:06:09.340288 (+1620028us) inbound_call.cc:173] Queueing success response
14:06:09.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
14:06:09.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:09.357833 1872 raft_consensus.cc:1273] T 00000000000000000000000000000000 P 4cb9c6adfa244ffd8ca9ad5d2c742023 [term 1 FOLLOWER]: Refusing update from remote peer 0f05cb838d9d41ee838751c0b5839531: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
14:06:09.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:09.358628 1872 rpcz_store.cc:275] Call kudu.consensus.ConsensusService.UpdateConsensus from 127.0.0.1:58710 (request call id 4) took 1534 ms. Trace:
14:06:09.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:09.358979 1872 rpcz_store.cc:276] 0902 14:06:07.823894 (+ 0us) service_pool.cc:168] Inserting onto call queue
14:06:09.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 14:06:07.824101 (+ 207us) service_pool.cc:225] Handling call
14:06:09.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 14:06:09.313572 (+1489471us) raft_consensus.cc:1469] Updating replica for 0 ops
14:06:09.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 14:06:09.358226 (+ 44654us) raft_consensus.cc:1695] Filling consensus response to leader.
14:06:09.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 14:06:09.358548 (+ 322us) inbound_call.cc:173] Queueing success response
14:06:09.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
14:06:09.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:09.362962 2000 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 0f05cb838d9d41ee838751c0b5839531 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4cb9c6adfa244ffd8ca9ad5d2c742023" member_type: VOTER last_known_addr { host: "127.1.172.189" port: 35607 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
14:06:09.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:09.504707 2063 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4cb9c6adfa244ffd8ca9ad5d2c742023 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "0f05cb838d9d41ee838751c0b5839531" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0f05cb838d9d41ee838751c0b5839531" member_type: VOTER last_known_addr { host: "127.1.172.190" port: 46329 } } peers { permanent_uuid: "4cb9c6adfa244ffd8ca9ad5d2c742023" member_type: VOTER last_known_addr { host: "127.1.172.189" port: 35607 } } peers { permanent_uuid: "0f503058e6a84256ab9186070c48408b" member_type: VOTER last_known_addr { host: "127.1.172.188" port: 36309 } } }
14:06:09.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:09.514503 2063 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4cb9c6adfa244ffd8ca9ad5d2c742023 [sys.catalog]: This master's current role is: FOLLOWER
14:06:09.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:09.513283 2053 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4cb9c6adfa244ffd8ca9ad5d2c742023 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 0f05cb838d9d41ee838751c0b5839531. Latest consensus state: current_term: 1 leader_uuid: "0f05cb838d9d41ee838751c0b5839531" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0f05cb838d9d41ee838751c0b5839531" member_type: VOTER last_known_addr { host: "127.1.172.190" port: 46329 } } peers { permanent_uuid: "4cb9c6adfa244ffd8ca9ad5d2c742023" member_type: VOTER last_known_addr { host: "127.1.172.189" port: 35607 } } peers { permanent_uuid: "0f503058e6a84256ab9186070c48408b" member_type: VOTER last_known_addr { host: "127.1.172.188" port: 36309 } } }
14:06:09.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:09.515762 2053 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4cb9c6adfa244ffd8ca9ad5d2c742023 [sys.catalog]: This master's current role is: FOLLOWER
14:06:09.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:09.585712 2070 catalog_manager.cc:1261] Loaded cluster ID: 0c29294b4d6c4be5afc6c01b4bfa41bd
14:06:09.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:09.586140 2070 catalog_manager.cc:1554] T 00000000000000000000000000000000 P 4cb9c6adfa244ffd8ca9ad5d2c742023: loading cluster ID for follower catalog manager: success
14:06:09.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:09.624409 2070 catalog_manager.cc:1576] T 00000000000000000000000000000000 P 4cb9c6adfa244ffd8ca9ad5d2c742023: acquiring CA information for follower catalog manager: success
14:06:09.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:09.644991 2070 catalog_manager.cc:1604] T 00000000000000000000000000000000 P 4cb9c6adfa244ffd8ca9ad5d2c742023: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
14:06:09.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:09.958381 2074 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0f05cb838d9d41ee838751c0b5839531 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "0f05cb838d9d41ee838751c0b5839531" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0f05cb838d9d41ee838751c0b5839531" member_type: VOTER last_known_addr { host: "127.1.172.190" port: 46329 } } peers { permanent_uuid: "4cb9c6adfa244ffd8ca9ad5d2c742023" member_type: VOTER last_known_addr { host: "127.1.172.189" port: 35607 } } peers { permanent_uuid: "0f503058e6a84256ab9186070c48408b" member_type: VOTER last_known_addr { host: "127.1.172.188" port: 36309 } } }
14:06:09.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:09.959756 2074 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0f05cb838d9d41ee838751c0b5839531 [sys.catalog]: This master's current role is: LEADER
14:06:11.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 14:06:11 dist-test-slave-m0vh krb5kdc[1726](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756821971, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.172.190@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:06:11.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:11.552744 1759 init.cc:295] Successfully reacquired a new kerberos TGT
14:06:12.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:10.675699 2049 debug-util.cc:398] Leaking SignalData structure 0x7b0800037b60 after lost signal to thread 2040
14:06:12.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:12.347496 2049 kernel_stack_watchdog.cc:198] Thread 2040 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 380ms:
14:06:12.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
14:06:12.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
14:06:12.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
14:06:12.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
14:06:12.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:09.297608 2051 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:06:12.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:12.349962 2040 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 3.058s user 0.784s sys 1.122s
14:06:12.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:12.350325 2040 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 3.058s user 0.784s sys 1.122s
14:06:12.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:12.366448 2040 server_base.cc:1047] running on GCE node
14:06:12.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:12.368330 2080 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:06:12.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:12.370087 2040 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:06:12.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:12.374302 2040 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:06:12.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:12.376103 2040 hybrid_clock.cc:648] HybridClock initialized: now 1756821972376039 us; error 57 us; skew 500 ppm
14:06:12.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:12.377295 2040 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:06:12.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 14:06:12 dist-test-slave-m0vh krb5kdc[1726](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756821972, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.172.129@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:06:12.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:12.391398 2040 init.cc:377] Logged in from keytab as kudu/127.1.172.129@KRBTEST.COM (short username kudu)
14:06:12.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:12.460114 2040 webserver.cc:480] Webserver started at http://127.1.172.129:45849/ using document root <none> and password file <none>
14:06:12.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:12.461549 2040 fs_manager.cc:362] Metadata directory not provided
14:06:12.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:12.461869 2040 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:06:12.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:12.462447 2040 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
14:06:12.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:12.481427 2040 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/ts-0/data/instance:
14:06:12.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "260dd313222a4c7ab6ac8af8728e736d"
14:06:12.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 14:06:12 on dist-test-slave-m0vh"
14:06:12.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:12.483104 2040 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/ts-0/wal/instance:
14:06:12.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "260dd313222a4c7ab6ac8af8728e736d"
14:06:12.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 14:06:12 on dist-test-slave-m0vh"
14:06:12.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:12.523519 2040 fs_manager.cc:696] Time spent creating directory manager: real 0.039s user 0.007s sys 0.005s
14:06:12.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:12.543217 2087 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:06:12.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:12.546715 2040 fs_manager.cc:730] Time spent opening block manager: real 0.018s user 0.005s sys 0.002s
14:06:12.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:12.547206 2040 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/ts-0/data,/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/ts-0/wal
14:06:12.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "260dd313222a4c7ab6ac8af8728e736d"
14:06:12.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 14:06:12 on dist-test-slave-m0vh"
14:06:12.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:12.547703 2040 fs_report.cc:389] FS layout report
14:06:12.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:06:12.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/ts-0/wal
14:06:12.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/ts-0/wal
14:06:12.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/ts-0/data/data
14:06:12.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:06:12.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:06:12.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:06:12.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:06:12.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:06:12.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:06:12.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:06:12.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:06:12.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:06:12.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:06:12.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:06:12.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:12.662990 2040 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:06:12.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:12.676177 2040 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:06:12.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:12.676873 2040 kserver.cc:163] Server-wide thread pool size limit: 3276
14:06:12.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:12.680775 2040 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
14:06:12.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:12.693161 2040 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
14:06:12.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:12.693485 2040 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
14:06:12.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:12.693819 2040 ts_tablet_manager.cc:610] Registered 0 tablets
14:06:12.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:12.694041 2040 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
14:06:13.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:13.208542 2040 rpc_server.cc:307] RPC server started. Bound to: 127.1.172.129:36281
14:06:13.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:13.208909 2202 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.1.172.129:36281 every 8 connection(s)
14:06:13.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:13.211660 2040 server_base.cc:1179] Dumped server information to /tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/ts-0/data/info.pb
14:06:13.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:13.219585 1714 external_mini_cluster.cc:1428] Started /tmp/dist-test-task50Kl3p/build/tsan/bin/kudu as pid 2040
14:06:13.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:13.220160 1714 external_mini_cluster.cc:1442] Reading /tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/ts-0/wal/instance
14:06:13.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 14:06:13 dist-test-slave-m0vh krb5kdc[1726](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756821972, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.172.129@KRBTEST.COM for kudu/127.1.172.189@KRBTEST.COM
14:06:13.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:13.329202 2203 heartbeater.cc:344] Connected to a master server at 127.1.172.189:35607
14:06:13.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:13.336144 2203 heartbeater.cc:461] Registering TS with master...
14:06:13.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.1.172.130@KRBTEST.COM; defaulting to no policy
14:06:13.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:13.338433 2203 heartbeater.cc:507] Master 127.1.172.189:35607 requested a full tablet report, sending...
14:06:13.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:06:13.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.1.172.130@KRBTEST.COM" created.
14:06:13.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:13.343863 1864 ts_manager.cc:194] Registered new tserver with Master: 260dd313222a4c7ab6ac8af8728e736d (127.1.172.129:36281)
14:06:13.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 14:06:13 dist-test-slave-m0vh krb5kdc[1726](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756821972, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.172.129@KRBTEST.COM for kudu/127.1.172.188@KRBTEST.COM
14:06:13.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 14:06:13 dist-test-slave-m0vh krb5kdc[1726](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756821972, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.172.129@KRBTEST.COM for kudu/127.1.172.190@KRBTEST.COM
14:06:13.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:13.366495 2204 heartbeater.cc:344] Connected to a master server at 127.1.172.188:36309
14:06:13.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:13.367031 2204 heartbeater.cc:461] Registering TS with master...
14:06:13.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:13.369519 2204 heartbeater.cc:507] Master 127.1.172.188:36309 requested a full tablet report, sending...
14:06:13.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:13.373672 1950 ts_manager.cc:194] Registered new tserver with Master: 260dd313222a4c7ab6ac8af8728e736d (127.1.172.129:36281)
14:06:13.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:13.375671 2209 heartbeater.cc:344] Connected to a master server at 127.1.172.190:46329
14:06:13.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:13.376171 2209 heartbeater.cc:461] Registering TS with master...
14:06:13.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:13.377597 2209 heartbeater.cc:507] Master 127.1.172.190:46329 requested a full tablet report, sending...
14:06:13.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:06:13.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.172.130 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/krb5kdc/kudu.keytab.
14:06:13.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.172.130 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/krb5kdc/kudu.keytab.
14:06:13.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:13.381072 1780 ts_manager.cc:194] Registered new tserver with Master: 260dd313222a4c7ab6ac8af8728e736d (127.1.172.129:36281)
14:06:13.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:13.381477 1714 mini_kdc.cc:279] Time spent creating service keytab for kudu/127.1.172.130: real 0.131s user 0.008s sys 0.016s
14:06:13.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:13.384927 1780 master_service.cc:496] Signed X509 certificate for tserver {username='kudu', principal='kudu/127.1.172.129@KRBTEST.COM'} at 127.1.172.129:58215
14:06:13.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.1.172.130@KRBTEST.COM; defaulting to no policy
14:06:13.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:06:13.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.1.172.130@KRBTEST.COM" created.
14:06:13.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:06:13.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.1.172.130 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/krb5kdc/kudu.keytab.
14:06:13.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.1.172.130 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/krb5kdc/kudu.keytab.
14:06:13.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:13.449321 1714 external_mini_cluster.cc:1366] Running /tmp/dist-test-task50Kl3p/build/tsan/bin/kudu
14:06:13.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task50Kl3p/build/tsan/bin/kudu
14:06:13.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/ts-1/wal
14:06:13.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/ts-1/data
14:06:13.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:06:13.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:06:13.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:06:13.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:06:13.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:06:13.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:06:13.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/ts-1/logs
14:06:13.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/ts-1/data/info.pb
14:06:13.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:06:13.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:06:13.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:06:13.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:06:13.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:06:13.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:06:13.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:06:13.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:06:13.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
14:06:13.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:06:13.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.172.130:0
14:06:13.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.172.130
14:06:13.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.172.130
14:06:13.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:06:13.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.172.190:46329,127.1.172.189:35607,127.1.172.188:36309
14:06:13.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/krb5kdc/kudu.keytab
14:06:13.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.172.130
14:06:13.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:06:13.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:06:13.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:06:13.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
14:06:13.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.172.148:42901
14:06:13.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:06:13.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:06:13.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:06:13.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false with env {KRB5CCNAME=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:06:13.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:13.953289 2218 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:06:13.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:13.954336 2218 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:06:13.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:13.955191 2218 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:06:13.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:13.994225 2218 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:06:13.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:13.995409 2218 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.1.172.130
14:06:14.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:14.041217 2218 tablet_server_runner.cc:78] Tablet server non-default flags:
14:06:14.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:06:14.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.172.148:42901
14:06:14.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:06:14.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:06:14.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/ts-1/data
14:06:14.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/ts-1/wal
14:06:14.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/krb5kdc/kudu.keytab
14:06:14.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.172.130
14:06:14.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:06:14.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:06:14.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.172.130:0
14:06:14.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:06:14.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:06:14.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:06:14.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:06:14.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:06:14.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:06:14.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/ts-1/data/info.pb
14:06:14.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.172.130
14:06:14.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:06:14.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.172.190:46329,127.1.172.189:35607,127.1.172.188:36309
14:06:14.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:06:14.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:06:14.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:06:14.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:06:14.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:06:14.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.172.130
14:06:14.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/ts-1/logs
14:06:14.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:06:14.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:06:14.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:06:14.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
14:06:14.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:06:14.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 06466b25e294b4d7d455e6c56263318dc34a80ef
14:06:14.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:06:14.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Sep 2025 13:43:21 UTC on 5fd53c4cbb9d
14:06:14.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7865
14:06:14.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:06:14.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:14.043108 2218 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:06:14.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:14.044456 2218 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:06:14.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:14.062776 2224 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:06:14.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 14:06:14 dist-test-slave-m0vh krb5kdc[1726](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756821974, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.172.189@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:06:14.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:14.212674 1841 init.cc:295] Successfully reacquired a new kerberos TGT
14:06:14.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:14.390165 2209 heartbeater.cc:499] Master 127.1.172.190:46329 was elected leader, sending a full tablet report...
14:06:15.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:15.470934 2223 debug-util.cc:398] Leaking SignalData structure 0x7b0800037b60 after lost signal to thread 2218
14:06:15.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:15.808550 2223 kernel_stack_watchdog.cc:198] Thread 2218 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 396ms:
14:06:15.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
14:06:15.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
14:06:15.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
14:06:15.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
14:06:15.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:14.066540 2225 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:06:15.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:15.811502 2218 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.743s user 0.491s sys 1.096s
14:06:15.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:15.811914 2218 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.745s user 0.491s sys 1.096s
14:06:15.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:15.813252 2227 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:06:15.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:15.815038 2226 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1744 milliseconds
14:06:15.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:15.815112 2218 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:06:15.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:15.816347 2218 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:06:15.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:15.818796 2218 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:06:15.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:15.820183 2218 hybrid_clock.cc:648] HybridClock initialized: now 1756821975820124 us; error 69 us; skew 500 ppm
14:06:15.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:15.821086 2218 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:06:15.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 14:06:15 dist-test-slave-m0vh krb5kdc[1726](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756821975, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.172.130@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:06:15.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:15.827809 2218 init.cc:377] Logged in from keytab as kudu/127.1.172.130@KRBTEST.COM (short username kudu)
14:06:15.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:15.832880 2218 webserver.cc:480] Webserver started at http://127.1.172.130:44203/ using document root <none> and password file <none>
14:06:15.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:15.833837 2218 fs_manager.cc:362] Metadata directory not provided
14:06:15.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:15.834048 2218 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:06:15.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:15.834479 2218 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
14:06:15.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:15.839484 2218 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/ts-1/data/instance:
14:06:15.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f61769ec2dbe465995cd5b0d0bbb69e6"
14:06:15.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 14:06:15 on dist-test-slave-m0vh"
14:06:15.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:15.840857 2218 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/ts-1/wal/instance:
14:06:15.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f61769ec2dbe465995cd5b0d0bbb69e6"
14:06:15.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 14:06:15 on dist-test-slave-m0vh"
14:06:15.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:15.849375 2218 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.003s sys 0.005s
14:06:15.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:15.856055 2242 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:06:15.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:15.857699 2218 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.002s sys 0.002s
14:06:15.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:15.858103 2218 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/ts-1/data,/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/ts-1/wal
14:06:15.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f61769ec2dbe465995cd5b0d0bbb69e6"
14:06:15.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 14:06:15 on dist-test-slave-m0vh"
14:06:15.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:15.858543 2218 fs_report.cc:389] FS layout report
14:06:15.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:06:15.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/ts-1/wal
14:06:15.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/ts-1/wal
14:06:15.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/ts-1/data/data
14:06:15.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:06:15.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:06:15.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:06:15.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:06:15.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:06:15.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:06:15.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:06:15.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:06:15.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:06:15.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:06:15.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:06:15.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:15.943756 2218 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:06:15.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:15.950264 2218 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:06:15.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:15.950693 2218 kserver.cc:163] Server-wide thread pool size limit: 3276
14:06:15.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:15.953298 2218 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
14:06:15.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:15.957708 2218 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
14:06:15.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:15.957926 2218 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
14:06:15.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:15.958117 2218 ts_tablet_manager.cc:610] Registered 0 tablets
14:06:15.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:15.958247 2218 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
14:06:16.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:16.340704 2218 rpc_server.cc:307] RPC server started. Bound to: 127.1.172.130:42843
14:06:16.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:16.340938 2354 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.1.172.130:42843 every 8 connection(s)
14:06:16.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:16.343753 2218 server_base.cc:1179] Dumped server information to /tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/ts-1/data/info.pb
14:06:16.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:16.353780 1714 external_mini_cluster.cc:1428] Started /tmp/dist-test-task50Kl3p/build/tsan/bin/kudu as pid 2218
14:06:16.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:16.354342 1714 external_mini_cluster.cc:1442] Reading /tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/ts-1/wal/instance
14:06:16.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.1.172.131@KRBTEST.COM; defaulting to no policy
14:06:16.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:06:16.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.1.172.131@KRBTEST.COM" created.
14:06:16.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 14:06:16 dist-test-slave-m0vh krb5kdc[1726](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756821975, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.172.130@KRBTEST.COM for kudu/127.1.172.190@KRBTEST.COM
14:06:16.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 14:06:16 dist-test-slave-m0vh krb5kdc[1726](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756821975, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.172.130@KRBTEST.COM for kudu/127.1.172.188@KRBTEST.COM
14:06:16.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 14:06:16 dist-test-slave-m0vh krb5kdc[1726](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756821975, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.172.130@KRBTEST.COM for kudu/127.1.172.189@KRBTEST.COM
14:06:16.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:06:16.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.172.131 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/krb5kdc/kudu.keytab.
14:06:16.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.172.131 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/krb5kdc/kudu.keytab.
14:06:16.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:16.483675 1714 mini_kdc.cc:279] Time spent creating service keytab for kudu/127.1.172.131: real 0.113s user 0.003s sys 0.023s
14:06:16.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:16.483955 2358 heartbeater.cc:344] Connected to a master server at 127.1.172.190:46329
14:06:16.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:16.484618 2358 heartbeater.cc:461] Registering TS with master...
14:06:16.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:16.486271 2358 heartbeater.cc:507] Master 127.1.172.190:46329 requested a full tablet report, sending...
14:06:16.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:16.489923 1780 ts_manager.cc:194] Registered new tserver with Master: f61769ec2dbe465995cd5b0d0bbb69e6 (127.1.172.130:42843)
14:06:16.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:16.493387 1780 master_service.cc:496] Signed X509 certificate for tserver {username='kudu', principal='kudu/127.1.172.130@KRBTEST.COM'} at 127.1.172.130:36367
14:06:16.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:16.493775 2355 heartbeater.cc:344] Connected to a master server at 127.1.172.189:35607
14:06:16.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:16.494225 2355 heartbeater.cc:461] Registering TS with master...
14:06:16.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:16.496170 2355 heartbeater.cc:507] Master 127.1.172.189:35607 requested a full tablet report, sending...
14:06:16.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:16.500224 1864 ts_manager.cc:194] Registered new tserver with Master: f61769ec2dbe465995cd5b0d0bbb69e6 (127.1.172.130:42843)
14:06:16.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:16.508917 2356 heartbeater.cc:344] Connected to a master server at 127.1.172.188:36309
14:06:16.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:16.509356 2356 heartbeater.cc:461] Registering TS with master...
14:06:16.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:16.510159 2356 heartbeater.cc:507] Master 127.1.172.188:36309 requested a full tablet report, sending...
14:06:16.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:16.514171 1950 ts_manager.cc:194] Registered new tserver with Master: f61769ec2dbe465995cd5b0d0bbb69e6 (127.1.172.130:42843)
14:06:16.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.1.172.131@KRBTEST.COM; defaulting to no policy
14:06:16.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:06:16.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.1.172.131@KRBTEST.COM" created.
14:06:16.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:06:16.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.1.172.131 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/krb5kdc/kudu.keytab.
14:06:16.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.1.172.131 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/krb5kdc/kudu.keytab.
14:06:16.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:16.551604 1714 external_mini_cluster.cc:1366] Running /tmp/dist-test-task50Kl3p/build/tsan/bin/kudu
14:06:16.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task50Kl3p/build/tsan/bin/kudu
14:06:16.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/ts-2/wal
14:06:16.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/ts-2/data
14:06:16.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:06:16.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:06:16.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:06:16.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:06:16.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:06:16.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:06:16.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/ts-2/logs
14:06:16.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/ts-2/data/info.pb
14:06:16.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:06:16.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:06:16.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:06:16.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:06:16.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:06:16.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:06:16.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:06:16.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:06:16.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
14:06:16.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:06:16.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.172.131:0
14:06:16.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.172.131
14:06:16.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.172.131
14:06:16.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:06:16.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.172.190:46329,127.1.172.189:35607,127.1.172.188:36309
14:06:16.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/krb5kdc/kudu.keytab
14:06:16.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.172.131
14:06:16.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:06:16.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:06:16.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:06:16.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
14:06:16.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.172.148:42901
14:06:16.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:06:16.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:06:16.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:06:16.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false with env {KRB5CCNAME=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:06:16.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:16.624949 1218 debug-util.cc:398] Leaking SignalData structure 0x7b08000b03a0 after lost signal to thread 1149
14:06:16.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:16.625818 1218 debug-util.cc:398] Leaking SignalData structure 0x7b0800083560 after lost signal to thread 1221
14:06:16.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 14:06:16 dist-test-slave-m0vh krb5kdc[1726](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756821976, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.172.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:06:16.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:16.696648 1928 init.cc:295] Successfully reacquired a new kerberos TGT
14:06:17.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:17.077100 2369 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:06:17.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:17.077888 2369 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:06:17.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:17.078979 2369 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:06:17.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:17.116580 2369 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:06:17.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:17.117762 2369 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.1.172.131
14:06:17.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:17.162959 2369 tablet_server_runner.cc:78] Tablet server non-default flags:
14:06:17.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:06:17.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.172.148:42901
14:06:17.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:06:17.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:06:17.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/ts-2/data
14:06:17.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/ts-2/wal
14:06:17.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/krb5kdc/kudu.keytab
14:06:17.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.172.131
14:06:17.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:06:17.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:06:17.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.172.131:0
14:06:17.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:06:17.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:06:17.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:06:17.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:06:17.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:06:17.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:06:17.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/ts-2/data/info.pb
14:06:17.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.172.131
14:06:17.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:06:17.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.172.190:46329,127.1.172.189:35607,127.1.172.188:36309
14:06:17.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:06:17.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:06:17.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:06:17.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:06:17.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:06:17.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.172.131
14:06:17.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/ts-2/logs
14:06:17.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:06:17.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:06:17.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:06:17.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
14:06:17.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:06:17.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 06466b25e294b4d7d455e6c56263318dc34a80ef
14:06:17.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:06:17.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Sep 2025 13:43:21 UTC on 5fd53c4cbb9d
14:06:17.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7865
14:06:17.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:06:17.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:17.165617 2369 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:06:17.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:17.167198 2369 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:06:17.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:17.190588 2376 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:06:17.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:17.501762 2358 heartbeater.cc:499] Master 127.1.172.190:46329 was elected leader, sending a full tablet report...
14:06:19.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:17.198820 2377 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:06:19.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:17.214745 2379 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:06:19.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:19.069293 2378 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1873 milliseconds
14:06:19.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:19.069438 2369 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:06:19.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:19.071043 2369 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:06:19.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:19.074524 2369 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:06:19.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:19.075992 2369 hybrid_clock.cc:648] HybridClock initialized: now 1756821979075842 us; error 158 us; skew 500 ppm
14:06:19.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:19.076925 2369 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:06:19.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 14:06:19 dist-test-slave-m0vh krb5kdc[1726](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756821979, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.172.131@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:06:19.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:19.083534 2369 init.cc:377] Logged in from keytab as kudu/127.1.172.131@KRBTEST.COM (short username kudu)
14:06:19.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:19.091140 2369 webserver.cc:480] Webserver started at http://127.1.172.131:43933/ using document root <none> and password file <none>
14:06:19.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:19.092528 2369 fs_manager.cc:362] Metadata directory not provided
14:06:19.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:19.092898 2369 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:06:19.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:19.093505 2369 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
14:06:19.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:19.100440 2369 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/ts-2/data/instance:
14:06:19.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fd93195deda54438b4dced1460052d02"
14:06:19.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 14:06:19 on dist-test-slave-m0vh"
14:06:19.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:19.102032 2369 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/ts-2/wal/instance:
14:06:19.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fd93195deda54438b4dced1460052d02"
14:06:19.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 14:06:19 on dist-test-slave-m0vh"
14:06:19.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:19.113215 2369 fs_manager.cc:696] Time spent creating directory manager: real 0.011s user 0.006s sys 0.006s
14:06:19.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:19.121078 2389 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:06:19.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:19.122946 2369 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.002s sys 0.003s
14:06:19.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:19.123577 2369 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/ts-2/data,/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/ts-2/wal
14:06:19.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fd93195deda54438b4dced1460052d02"
14:06:19.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 14:06:19 on dist-test-slave-m0vh"
14:06:19.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:19.124356 2369 fs_report.cc:389] FS layout report
14:06:19.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:06:19.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/ts-2/wal
14:06:19.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/ts-2/wal
14:06:19.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/ts-2/data/data
14:06:19.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:06:19.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:06:19.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:06:19.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:06:19.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:06:19.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:06:19.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:06:19.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:06:19.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:06:19.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:06:19.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:06:19.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:19.184904 2369 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:06:19.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:19.195351 2369 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:06:19.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:19.195814 2369 kserver.cc:163] Server-wide thread pool size limit: 3276
14:06:19.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:19.200493 2369 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
14:06:19.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:19.206971 2369 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
14:06:19.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:19.207262 2369 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
14:06:19.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:19.207577 2369 ts_tablet_manager.cc:610] Registered 0 tablets
14:06:19.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:19.207789 2369 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
14:06:19.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:19.591068 2369 rpc_server.cc:307] RPC server started. Bound to: 127.1.172.131:36021
14:06:19.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:19.591552 2501 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.1.172.131:36021 every 8 connection(s)
14:06:19.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:19.594372 2369 server_base.cc:1179] Dumped server information to /tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/ts-2/data/info.pb
14:06:19.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:19.599650 1714 external_mini_cluster.cc:1428] Started /tmp/dist-test-task50Kl3p/build/tsan/bin/kudu as pid 2369
14:06:19.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:19.601977 1714 external_mini_cluster.cc:1442] Reading /tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/ts-2/wal/instance
14:06:19.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 14:06:19 dist-test-slave-m0vh krb5kdc[1726](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756821979, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.172.131@KRBTEST.COM for kudu/127.1.172.189@KRBTEST.COM
14:06:19.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 14:06:19 dist-test-slave-m0vh krb5kdc[1726](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756821979, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.172.131@KRBTEST.COM for kudu/127.1.172.190@KRBTEST.COM
14:06:19.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 14:06:19 dist-test-slave-m0vh krb5kdc[1726](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756821979, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.172.131@KRBTEST.COM for kudu/127.1.172.188@KRBTEST.COM
14:06:19.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:19.720520 2505 heartbeater.cc:344] Connected to a master server at 127.1.172.190:46329
14:06:19.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:19.721290 2505 heartbeater.cc:461] Registering TS with master...
14:06:19.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:19.723841 2505 heartbeater.cc:507] Master 127.1.172.190:46329 requested a full tablet report, sending...
14:06:19.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:19.725605 2502 heartbeater.cc:344] Connected to a master server at 127.1.172.189:35607
14:06:19.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:19.726260 2502 heartbeater.cc:461] Registering TS with master...
14:06:19.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:19.728145 2502 heartbeater.cc:507] Master 127.1.172.189:35607 requested a full tablet report, sending...
14:06:19.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:19.728490 1779 ts_manager.cc:194] Registered new tserver with Master: fd93195deda54438b4dced1460052d02 (127.1.172.131:36021)
14:06:19.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:19.731794 1779 master_service.cc:496] Signed X509 certificate for tserver {username='kudu', principal='kudu/127.1.172.131@KRBTEST.COM'} at 127.1.172.131:44229
14:06:19.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:19.736349 1863 ts_manager.cc:194] Registered new tserver with Master: fd93195deda54438b4dced1460052d02 (127.1.172.131:36021)
14:06:19.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:19.743809 2503 heartbeater.cc:344] Connected to a master server at 127.1.172.188:36309
14:06:19.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:19.744248 2503 heartbeater.cc:461] Registering TS with master...
14:06:19.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:19.745234 2503 heartbeater.cc:507] Master 127.1.172.188:36309 requested a full tablet report, sending...
14:06:19.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:19.748733 1950 ts_manager.cc:194] Registered new tserver with Master: fd93195deda54438b4dced1460052d02 (127.1.172.131:36021)
14:06:19.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:19.759891 1714 external_mini_cluster.cc:949] 3 TS(s) registered with all masters
14:06:19.782 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:06:19.784 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
14:06:19.796 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
14:06:19.797 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
14:06:19.800 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.1.172.190"
port: 46329
}
bound_http_address {
host: "127.1.172.190"
port: 46345
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.1.172.189"
port: 35607
}
bound_http_address {
host: "127.1.172.189"
port: 33923
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.1.172.188"
port: 36309
}
bound_http_address {
host: "127.1.172.188"
port: 37159
}
}
}
14:06:19.800 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
14:06:19.802 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.1.172.129"
port: 36281
}
bound_http_address {
host: "127.1.172.129"
port: 45849
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.1.172.130"
port: 42843
}
bound_http_address {
host: "127.1.172.130"
port: 44203
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.1.172.131"
port: 36021
}
bound_http_address {
host: "127.1.172.131"
port: 43933
}
}
}
14:06:19.802 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
14:06:19.804 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
14:06:19.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 14:06:19 dist-test-slave-m0vh krb5kdc[1726](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756821979, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:06:19.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
14:06:19.856 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:06:19.859 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/krb5kdc/krb5cc
14:06:19.869 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
14:06:19.870 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/krb5kdc/krb5cc
14:06:19.872 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
14:06:19.877 [DEBUG - Time-limited test] (SecurityContext.java:158) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
14:06:19.879 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
14:06:19.880 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
14:06:19.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 14:06:19 dist-test-slave-m0vh krb5kdc[1726](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756821979, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:06:19.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
14:06:19.935 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:06:19.936 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/krb5kdc/krb5cc
14:06:19.937 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
14:06:19.938 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.1.172.188:36309
14:06:19.940 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
14:06:19.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:19.941004 1714 external_mini_cluster.cc:1658] Killing /tmp/dist-test-task50Kl3p/build/tsan/bin/kudu with pid 1910
14:06:19.971 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:06:19.972 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.1.172.190:46329
14:06:19.972 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
14:06:19.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:19.972671 1714 external_mini_cluster.cc:1658] Killing /tmp/dist-test-task50Kl3p/build/tsan/bin/kudu with pid 1743
14:06:20.001 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:06:20.001 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.1.172.189:35607
14:06:20.002 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
14:06:20.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:20.002431 1714 external_mini_cluster.cc:1658] Killing /tmp/dist-test-task50Kl3p/build/tsan/bin/kudu with pid 1824
14:06:20.035 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:06:20.036 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.1.172.188:36309
14:06:20.041 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
14:06:20.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:20.043509 1714 external_mini_cluster.cc:1366] Running /tmp/dist-test-task50Kl3p/build/tsan/bin/kudu
14:06:20.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task50Kl3p/build/tsan/bin/kudu
14:06:20.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/master-2/wal
14:06:20.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/master-2/data
14:06:20.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:06:20.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:06:20.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:06:20.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:06:20.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:06:20.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:06:20.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/master-2/logs
14:06:20.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/master-2/data/info.pb
14:06:20.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:06:20.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:06:20.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:06:20.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:06:20.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:06:20.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:06:20.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:06:20.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:06:20.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:06:20.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:06:20.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:06:20.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:06:20.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.172.188:36309
14:06:20.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.172.188
14:06:20.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37159
14:06:20.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/krb5kdc/kudu.keytab
14:06:20.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.172.188
14:06:20.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:06:20.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:06:20.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:06:20.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
14:06:20.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.172.148:42901
14:06:20.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:06:20.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:06:20.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:06:20.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:06:20.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.172.190:46329,127.1.172.189:35607,127.1.172.188:36309
14:06:20.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:06:20.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37243/jwks.json
14:06:20.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
14:06:20.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:06:20.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:20.389144 2203 heartbeater.cc:646] Failed to heartbeat to 127.1.172.189:35607 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.1.172.189:35607: connect: Connection refused (error 111)
14:06:20.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:20.405582 2517 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:06:20.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:20.406164 2517 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:06:20.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:20.406821 2517 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:06:20.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:20.443467 2517 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
14:06:20.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:20.443856 2517 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:06:20.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:20.444336 2517 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:06:20.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:20.444737 2517 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:06:20.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:20.445048 2517 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:06:20.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:20.445245 2517 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:06:20.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:20.445389 2517 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37243/jwks.json
14:06:20.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:20.486704 2517 master_runner.cc:386] Master server non-default flags:
14:06:20.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:06:20.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.172.148:42901
14:06:20.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:06:20.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:06:20.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/master-2/data
14:06:20.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/master-2/wal
14:06:20.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
14:06:20.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:06:20.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.172.190:46329,127.1.172.189:35607,127.1.172.188:36309
14:06:20.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/krb5kdc/kudu.keytab
14:06:20.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.172.188
14:06:20.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:06:20.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:06:20.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:06:20.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.172.188:36309
14:06:20.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:06:20.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:06:20.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:06:20.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37243/jwks.json
14:06:20.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:06:20.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:06:20.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:06:20.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:06:20.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:06:20.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/master-2/data/info.pb
14:06:20.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.172.188
14:06:20.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37159
14:06:20.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:06:20.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:06:20.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:06:20.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:06:20.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:06:20.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/master-2/logs
14:06:20.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:06:20.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:06:20.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:06:20.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:06:20.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:06:20.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 06466b25e294b4d7d455e6c56263318dc34a80ef
14:06:20.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:06:20.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Sep 2025 13:43:21 UTC on 5fd53c4cbb9d
14:06:20.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7865
14:06:20.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:06:20.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:20.488118 2517 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:06:20.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:20.489171 2517 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:06:20.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:20.501082 2525 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:06:20.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:20.541946 2355 heartbeater.cc:646] Failed to heartbeat to 127.1.172.189:35607 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.1.172.189:35607: connect: Connection refused (error 111)
14:06:20.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:20.738410 2505 heartbeater.cc:499] Master 127.1.172.190:46329 was elected leader, sending a full tablet report...
14:06:20.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:20.768607 2502 heartbeater.cc:646] Failed to heartbeat to 127.1.172.189:35607 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.1.172.189:35607: connect: Connection refused (error 111)
14:06:22.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:22.328935 2498 debug-util.cc:398] Leaking SignalData structure 0x7b08000aec80 after lost signal to thread 2371
14:06:22.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:22.330549 2498 debug-util.cc:398] Leaking SignalData structure 0x7b08000b5aa0 after lost signal to thread 2501
14:06:22.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 14:06:22 dist-test-slave-m0vh krb5kdc[1726](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756821982, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.172.129@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:06:22.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:22.432893 2083 init.cc:295] Successfully reacquired a new kerberos TGT
14:06:23.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:20.502156 2526 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:06:23.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:21.907827 2524 debug-util.cc:398] Leaking SignalData structure 0x7b0800037200 after lost signal to thread 2517
14:06:23.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:23.035794 2517 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 2.534s user 0.872s sys 1.550s
14:06:23.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:23.037525 2527 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2533 milliseconds
14:06:23.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:23.038498 2528 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:06:23.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:23.037801 2517 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 2.536s user 0.872s sys 1.551s
14:06:23.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:23.039173 2517 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:06:23.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:23.043850 2517 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:06:23.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:23.047233 2517 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:06:23.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:23.049422 2517 hybrid_clock.cc:648] HybridClock initialized: now 1756821983049791 us; error 479 us; skew 500 ppm
14:06:23.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:23.050351 2517 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:06:23.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 14:06:23 dist-test-slave-m0vh krb5kdc[1726](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756821983, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.172.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:06:23.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:23.059159 2517 init.cc:377] Logged in from keytab as kudu/127.1.172.188@KRBTEST.COM (short username kudu)
14:06:23.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:23.068328 2517 webserver.cc:480] Webserver started at http://127.1.172.188:37159/ using document root <none> and password file <none>
14:06:23.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:23.069425 2517 fs_manager.cc:362] Metadata directory not provided
14:06:23.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:23.069690 2517 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:06:23.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:23.082386 2517 fs_manager.cc:714] Time spent opening directory manager: real 0.008s user 0.005s sys 0.005s
14:06:23.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:23.088171 2560 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:06:23.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:23.089664 2517 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.002s sys 0.002s
14:06:23.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:23.090144 2517 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/master-2/data,/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/master-2/wal
14:06:23.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0f503058e6a84256ab9186070c48408b"
14:06:23.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 14:06:06 on dist-test-slave-m0vh"
14:06:23.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:23.093410 2517 fs_report.cc:389] FS layout report
14:06:23.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:06:23.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/master-2/wal
14:06:23.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/master-2/wal
14:06:23.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/master-2/data/data
14:06:23.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:06:23.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:06:23.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:06:23.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:06:23.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:06:23.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:06:23.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:06:23.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:06:23.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:06:23.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:06:23.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:06:23.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:23.197708 2517 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:06:23.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:23.206583 2517 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:06:23.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:23.207226 2517 kserver.cc:163] Server-wide thread pool size limit: 3276
14:06:23.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:23.324957 2517 rpc_server.cc:307] RPC server started. Bound to: 127.1.172.188:36309
14:06:23.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:23.325016 2611 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.1.172.188:36309 every 8 connection(s)
14:06:23.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:23.328267 2517 server_base.cc:1179] Dumped server information to /tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/master-2/data/info.pb
14:06:23.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:23.335268 1714 external_mini_cluster.cc:1428] Started /tmp/dist-test-task50Kl3p/build/tsan/bin/kudu as pid 2517
14:06:23.336 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:06:23.336 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.1.172.190:46329
14:06:23.337 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
14:06:23.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:23.341723 1714 external_mini_cluster.cc:1366] Running /tmp/dist-test-task50Kl3p/build/tsan/bin/kudu
14:06:23.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task50Kl3p/build/tsan/bin/kudu
14:06:23.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/master-0/wal
14:06:23.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/master-0/data
14:06:23.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:06:23.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:06:23.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:06:23.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:06:23.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:06:23.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:06:23.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/master-0/logs
14:06:23.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/master-0/data/info.pb
14:06:23.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:06:23.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:06:23.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:06:23.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:06:23.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:06:23.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:06:23.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:06:23.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:06:23.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:06:23.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:06:23.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:06:23.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:06:23.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.172.190:46329
14:06:23.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.172.190
14:06:23.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46345
14:06:23.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/krb5kdc/kudu.keytab
14:06:23.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.172.190
14:06:23.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:06:23.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:06:23.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:06:23.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
14:06:23.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.172.148:42901
14:06:23.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:06:23.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:06:23.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:06:23.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:06:23.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.172.190:46329,127.1.172.189:35607,127.1.172.188:36309
14:06:23.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:06:23.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37243/jwks.json
14:06:23.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
14:06:23.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:06:23.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:23.356380 2612 sys_catalog.cc:263] Verifying existing consensus state
14:06:23.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:23.362471 2612 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 0f503058e6a84256ab9186070c48408b: Bootstrap starting.
14:06:23.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:23.411708 2612 log.cc:826] T 00000000000000000000000000000000 P 0f503058e6a84256ab9186070c48408b: Log is configured to *not* fsync() on all Append() calls
14:06:23.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:23.433655 2612 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 0f503058e6a84256ab9186070c48408b: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
14:06:23.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:23.434888 2612 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 0f503058e6a84256ab9186070c48408b: Bootstrap complete.
14:06:23.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:23.492916 2612 raft_consensus.cc:357] T 00000000000000000000000000000000 P 0f503058e6a84256ab9186070c48408b [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0f05cb838d9d41ee838751c0b5839531" member_type: VOTER last_known_addr { host: "127.1.172.190" port: 46329 } } peers { permanent_uuid: "4cb9c6adfa244ffd8ca9ad5d2c742023" member_type: VOTER last_known_addr { host: "127.1.172.189" port: 35607 } } peers { permanent_uuid: "0f503058e6a84256ab9186070c48408b" member_type: VOTER last_known_addr { host: "127.1.172.188" port: 36309 } }
14:06:23.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:23.497875 2612 raft_consensus.cc:738] T 00000000000000000000000000000000 P 0f503058e6a84256ab9186070c48408b [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 0f503058e6a84256ab9186070c48408b, State: Initialized, Role: FOLLOWER
14:06:23.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 14:06:23 dist-test-slave-m0vh krb5kdc[1726](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756821982, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.172.129@KRBTEST.COM for kudu/127.1.172.188@KRBTEST.COM
14:06:23.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:23.498986 2612 consensus_queue.cc:260] T 00000000000000000000000000000000 P 0f503058e6a84256ab9186070c48408b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0f05cb838d9d41ee838751c0b5839531" member_type: VOTER last_known_addr { host: "127.1.172.190" port: 46329 } } peers { permanent_uuid: "4cb9c6adfa244ffd8ca9ad5d2c742023" member_type: VOTER last_known_addr { host: "127.1.172.189" port: 35607 } } peers { permanent_uuid: "0f503058e6a84256ab9186070c48408b" member_type: VOTER last_known_addr { host: "127.1.172.188" port: 36309 } }
14:06:23.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:23.516458 2621 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0f503058e6a84256ab9186070c48408b [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0f05cb838d9d41ee838751c0b5839531" member_type: VOTER last_known_addr { host: "127.1.172.190" port: 46329 } } peers { permanent_uuid: "4cb9c6adfa244ffd8ca9ad5d2c742023" member_type: VOTER last_known_addr { host: "127.1.172.189" port: 35607 } } peers { permanent_uuid: "0f503058e6a84256ab9186070c48408b" member_type: VOTER last_known_addr { host: "127.1.172.188" port: 36309 } } }
14:06:23.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:23.517391 2621 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0f503058e6a84256ab9186070c48408b [sys.catalog]: This master's current role is: FOLLOWER
14:06:23.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:23.519145 2204 heartbeater.cc:344] Connected to a master server at 127.1.172.188:36309
14:06:23.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:23.519948 2612 sys_catalog.cc:564] T 00000000000000000000000000000000 P 0f503058e6a84256ab9186070c48408b [sys.catalog]: configured and running, proceeding with master startup.
14:06:23.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:23.593286 2632 catalog_manager.cc:1261] Loaded cluster ID: 0c29294b4d6c4be5afc6c01b4bfa41bd
14:06:23.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:23.593935 2632 catalog_manager.cc:1554] T 00000000000000000000000000000000 P 0f503058e6a84256ab9186070c48408b: loading cluster ID for follower catalog manager: success
14:06:23.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:23.602533 2632 catalog_manager.cc:1576] T 00000000000000000000000000000000 P 0f503058e6a84256ab9186070c48408b: acquiring CA information for follower catalog manager: success
14:06:23.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:23.609760 2632 catalog_manager.cc:1604] T 00000000000000000000000000000000 P 0f503058e6a84256ab9186070c48408b: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
14:06:23.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:23.597719 2517 master_runner.cc:186] Error getting master registration for 127.1.172.190:46329: Network error: Client connection negotiation failed: client connection to 127.1.172.190:46329: connect: Connection refused (error 111),
14:06:23.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:23.628759 2517 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:06:23.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:23.646842 2356 heartbeater.cc:344] Connected to a master server at 127.1.172.188:36309
14:06:23.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:23.654724 2577 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "f61769ec2dbe465995cd5b0d0bbb69e6" instance_seqno: 1756821976247767) as {username='kudu', principal='kudu/127.1.172.130@KRBTEST.COM'} at 127.1.172.130:46293; Asking this server to re-register.
14:06:23.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:23.656839 2356 heartbeater.cc:461] Registering TS with master...
14:06:23.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:23.661298 2577 ts_manager.cc:194] Registered new tserver with Master: f61769ec2dbe465995cd5b0d0bbb69e6 (127.1.172.130:42843)
14:06:23.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:23.909788 2503 heartbeater.cc:344] Connected to a master server at 127.1.172.188:36309
14:06:23.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:23.913444 2577 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "fd93195deda54438b4dced1460052d02" instance_seqno: 1756821979496130) as {username='kudu', principal='kudu/127.1.172.131@KRBTEST.COM'} at 127.1.172.131:48287; Asking this server to re-register.
14:06:23.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:23.915441 2503 heartbeater.cc:461] Registering TS with master...
14:06:23.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:23.920663 2577 ts_manager.cc:194] Registered new tserver with Master: fd93195deda54438b4dced1460052d02 (127.1.172.131:36021)
14:06:23.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:23.973049 2614 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:06:23.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:23.973743 2614 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:06:23.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:23.974519 2614 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:06:24.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:24.035264 2614 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
14:06:24.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:24.035670 2614 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:06:24.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:24.036176 2614 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:06:24.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:24.036556 2614 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:06:24.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:24.036926 2614 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:06:24.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:24.037284 2614 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:06:24.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:24.037570 2614 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37243/jwks.json
14:06:24.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:24.091013 2614 master_runner.cc:386] Master server non-default flags:
14:06:24.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:06:24.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.172.148:42901
14:06:24.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:06:24.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:06:24.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/master-0/data
14:06:24.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/master-0/wal
14:06:24.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
14:06:24.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:06:24.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.172.190:46329,127.1.172.189:35607,127.1.172.188:36309
14:06:24.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/krb5kdc/kudu.keytab
14:06:24.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.172.190
14:06:24.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:06:24.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:06:24.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:06:24.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.172.190:46329
14:06:24.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:06:24.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:06:24.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:06:24.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37243/jwks.json
14:06:24.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:06:24.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:06:24.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:06:24.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:06:24.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:06:24.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/master-0/data/info.pb
14:06:24.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.172.190
14:06:24.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46345
14:06:24.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:06:24.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:06:24.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:06:24.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:06:24.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:06:24.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/master-0/logs
14:06:24.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:06:24.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:06:24.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:06:24.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:06:24.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:06:24.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 06466b25e294b4d7d455e6c56263318dc34a80ef
14:06:24.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:06:24.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Sep 2025 13:43:21 UTC on 5fd53c4cbb9d
14:06:24.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7865
14:06:24.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:06:24.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:24.092835 2614 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:06:24.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:24.094311 2614 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:06:24.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:24.113670 2647 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:06:24.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:24.526305 2577 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "260dd313222a4c7ab6ac8af8728e736d" instance_seqno: 1756821973100173) as {username='kudu', principal='kudu/127.1.172.129@KRBTEST.COM'} at 127.1.172.129:42551; Asking this server to re-register.
14:06:24.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:24.528708 2204 heartbeater.cc:461] Registering TS with master...
14:06:24.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:24.532861 2577 ts_manager.cc:194] Registered new tserver with Master: 260dd313222a4c7ab6ac8af8728e736d (127.1.172.129:36281)
14:06:25.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:25.124929 2659 raft_consensus.cc:491] T 00000000000000000000000000000000 P 0f503058e6a84256ab9186070c48408b [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:06:25.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:25.126240 2659 raft_consensus.cc:513] T 00000000000000000000000000000000 P 0f503058e6a84256ab9186070c48408b [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0f05cb838d9d41ee838751c0b5839531" member_type: VOTER last_known_addr { host: "127.1.172.190" port: 46329 } } peers { permanent_uuid: "4cb9c6adfa244ffd8ca9ad5d2c742023" member_type: VOTER last_known_addr { host: "127.1.172.189" port: 35607 } } peers { permanent_uuid: "0f503058e6a84256ab9186070c48408b" member_type: VOTER last_known_addr { host: "127.1.172.188" port: 36309 } }
14:06:25.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:25.153887 2659 leader_election.cc:290] T 00000000000000000000000000000000 P 0f503058e6a84256ab9186070c48408b [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 0f05cb838d9d41ee838751c0b5839531 (127.1.172.190:46329), 4cb9c6adfa244ffd8ca9ad5d2c742023 (127.1.172.189:35607)
14:06:25.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:25.167061 2564 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.172.190:46329: connect: Connection refused (error 111)
14:06:25.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:25.195590 2562 leader_election.cc:336] T 00000000000000000000000000000000 P 0f503058e6a84256ab9186070c48408b [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 4cb9c6adfa244ffd8ca9ad5d2c742023 (127.1.172.189:35607): Network error: Client connection negotiation failed: client connection to 127.1.172.189:35607: connect: Connection refused (error 111)
14:06:25.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:25.196732 2564 leader_election.cc:336] T 00000000000000000000000000000000 P 0f503058e6a84256ab9186070c48408b [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 0f05cb838d9d41ee838751c0b5839531 (127.1.172.190:46329): Network error: Client connection negotiation failed: client connection to 127.1.172.190:46329: connect: Connection refused (error 111)
14:06:25.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:25.197309 2564 leader_election.cc:304] T 00000000000000000000000000000000 P 0f503058e6a84256ab9186070c48408b [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 0f503058e6a84256ab9186070c48408b; no voters: 0f05cb838d9d41ee838751c0b5839531, 4cb9c6adfa244ffd8ca9ad5d2c742023
14:06:25.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:25.198534 2659 raft_consensus.cc:2747] T 00000000000000000000000000000000 P 0f503058e6a84256ab9186070c48408b [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
14:06:25.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 14:06:25 dist-test-slave-m0vh krb5kdc[1726](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756821985, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.172.130@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:06:25.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:25.833171 2238 init.cc:295] Successfully reacquired a new kerberos TGT
14:06:25.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:25.508951 2646 debug-util.cc:398] Leaking SignalData structure 0x7b0800037200 after lost signal to thread 2614
14:06:25.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:25.955046 2614 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.834s user 0.002s sys 0.003s
14:06:25.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:25.956874 2614 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.836s user 0.002s sys 0.003s
14:06:25.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:24.119823 2648 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:06:25.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:25.959458 2650 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:06:25.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:25.967307 2614 server_base.cc:1047] running on GCE node
14:06:25.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:25.968873 2614 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:06:25.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:25.972283 2614 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:06:25.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:25.973745 2614 hybrid_clock.cc:648] HybridClock initialized: now 1756821985973626 us; error 116 us; skew 500 ppm
14:06:25.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:25.975050 2614 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:06:25.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 14:06:25 dist-test-slave-m0vh krb5kdc[1726](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756821985, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.172.190@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:06:25.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:25.983131 2614 init.cc:377] Logged in from keytab as kudu/127.1.172.190@KRBTEST.COM (short username kudu)
14:06:25.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:25.989018 2614 webserver.cc:480] Webserver started at http://127.1.172.190:46345/ using document root <none> and password file <none>
14:06:25.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:25.990094 2614 fs_manager.cc:362] Metadata directory not provided
14:06:25.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:25.990355 2614 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:06:26.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:25.999632 2614 fs_manager.cc:714] Time spent opening directory manager: real 0.006s user 0.003s sys 0.004s
14:06:26.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:26.005556 2675 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:06:26.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:26.006860 2614 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.006s sys 0.000s
14:06:26.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:26.007218 2614 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/master-0/data,/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/master-0/wal
14:06:26.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0f05cb838d9d41ee838751c0b5839531"
14:06:26.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 14:06:01 on dist-test-slave-m0vh"
14:06:26.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:26.009301 2614 fs_report.cc:389] FS layout report
14:06:26.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:06:26.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/master-0/wal
14:06:26.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/master-0/wal
14:06:26.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/master-0/data/data
14:06:26.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:06:26.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:06:26.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:06:26.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:06:26.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:06:26.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:06:26.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:06:26.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:06:26.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:06:26.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:06:26.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:06:26.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:26.087611 2614 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:06:26.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:26.099077 2614 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:06:26.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:26.099740 2614 kserver.cc:163] Server-wide thread pool size limit: 3276
14:06:26.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:26.197088 2614 rpc_server.cc:307] RPC server started. Bound to: 127.1.172.190:46329
14:06:26.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:26.197139 2726 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.1.172.190:46329 every 8 connection(s)
14:06:26.207 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:06:26.207 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.1.172.189:35607
14:06:26.208 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
14:06:26.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:26.201011 2614 server_base.cc:1179] Dumped server information to /tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/master-0/data/info.pb
14:06:26.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:26.205374 1714 external_mini_cluster.cc:1428] Started /tmp/dist-test-task50Kl3p/build/tsan/bin/kudu as pid 2614
14:06:26.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:26.222175 1714 external_mini_cluster.cc:1366] Running /tmp/dist-test-task50Kl3p/build/tsan/bin/kudu
14:06:26.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task50Kl3p/build/tsan/bin/kudu
14:06:26.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/master-1/wal
14:06:26.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/master-1/data
14:06:26.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:06:26.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:06:26.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:06:26.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:06:26.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:06:26.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:06:26.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/master-1/logs
14:06:26.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/master-1/data/info.pb
14:06:26.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:06:26.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:06:26.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:06:26.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:06:26.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:06:26.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:06:26.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:06:26.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:06:26.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:06:26.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:06:26.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:06:26.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:06:26.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.172.189:35607
14:06:26.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.172.189
14:06:26.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33923
14:06:26.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/krb5kdc/kudu.keytab
14:06:26.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.172.189
14:06:26.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:06:26.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:06:26.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:06:26.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
14:06:26.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.172.148:42901
14:06:26.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:06:26.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:06:26.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:06:26.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:06:26.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.172.190:46329,127.1.172.189:35607,127.1.172.188:36309
14:06:26.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:06:26.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37243/jwks.json
14:06:26.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
14:06:26.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:06:26.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:26.231886 2727 sys_catalog.cc:263] Verifying existing consensus state
14:06:26.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:26.240844 2727 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 0f05cb838d9d41ee838751c0b5839531: Bootstrap starting.
14:06:26.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:26.299254 2727 log.cc:826] T 00000000000000000000000000000000 P 0f05cb838d9d41ee838751c0b5839531: Log is configured to *not* fsync() on all Append() calls
14:06:26.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:26.321815 2727 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 0f05cb838d9d41ee838751c0b5839531: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
14:06:26.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:26.323022 2727 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 0f05cb838d9d41ee838751c0b5839531: Bootstrap complete.
14:06:26.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:26.376576 2727 raft_consensus.cc:357] T 00000000000000000000000000000000 P 0f05cb838d9d41ee838751c0b5839531 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0f05cb838d9d41ee838751c0b5839531" member_type: VOTER last_known_addr { host: "127.1.172.190" port: 46329 } } peers { permanent_uuid: "4cb9c6adfa244ffd8ca9ad5d2c742023" member_type: VOTER last_known_addr { host: "127.1.172.189" port: 35607 } } peers { permanent_uuid: "0f503058e6a84256ab9186070c48408b" member_type: VOTER last_known_addr { host: "127.1.172.188" port: 36309 } }
14:06:26.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:26.380072 2727 raft_consensus.cc:738] T 00000000000000000000000000000000 P 0f05cb838d9d41ee838751c0b5839531 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 0f05cb838d9d41ee838751c0b5839531, State: Initialized, Role: FOLLOWER
14:06:26.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:26.381119 2727 consensus_queue.cc:260] T 00000000000000000000000000000000 P 0f05cb838d9d41ee838751c0b5839531 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0f05cb838d9d41ee838751c0b5839531" member_type: VOTER last_known_addr { host: "127.1.172.190" port: 46329 } } peers { permanent_uuid: "4cb9c6adfa244ffd8ca9ad5d2c742023" member_type: VOTER last_known_addr { host: "127.1.172.189" port: 35607 } } peers { permanent_uuid: "0f503058e6a84256ab9186070c48408b" member_type: VOTER last_known_addr { host: "127.1.172.188" port: 36309 } }
14:06:26.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:26.384270 2731 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0f05cb838d9d41ee838751c0b5839531 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0f05cb838d9d41ee838751c0b5839531" member_type: VOTER last_known_addr { host: "127.1.172.190" port: 46329 } } peers { permanent_uuid: "4cb9c6adfa244ffd8ca9ad5d2c742023" member_type: VOTER last_known_addr { host: "127.1.172.189" port: 35607 } } peers { permanent_uuid: "0f503058e6a84256ab9186070c48408b" member_type: VOTER last_known_addr { host: "127.1.172.188" port: 36309 } } }
14:06:26.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:26.386258 2727 sys_catalog.cc:564] T 00000000000000000000000000000000 P 0f05cb838d9d41ee838751c0b5839531 [sys.catalog]: configured and running, proceeding with master startup.
14:06:26.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:26.386646 2731 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0f05cb838d9d41ee838751c0b5839531 [sys.catalog]: This master's current role is: FOLLOWER
14:06:26.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:26.441900 2742 catalog_manager.cc:1261] Loaded cluster ID: 0c29294b4d6c4be5afc6c01b4bfa41bd
14:06:26.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:26.442193 2742 catalog_manager.cc:1554] T 00000000000000000000000000000000 P 0f05cb838d9d41ee838751c0b5839531: loading cluster ID for follower catalog manager: success
14:06:26.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 14:06:26 dist-test-slave-m0vh krb5kdc[1726](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756821985, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.172.190@KRBTEST.COM for kudu/127.1.172.190@KRBTEST.COM
14:06:26.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:26.460345 2742 catalog_manager.cc:1576] T 00000000000000000000000000000000 P 0f05cb838d9d41ee838751c0b5839531: acquiring CA information for follower catalog manager: success
14:06:26.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:26.471351 2742 catalog_manager.cc:1604] T 00000000000000000000000000000000 P 0f05cb838d9d41ee838751c0b5839531: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
14:06:26.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:26.550752 2209 heartbeater.cc:344] Connected to a master server at 127.1.172.190:46329
14:06:26.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:26.559720 2692 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "260dd313222a4c7ab6ac8af8728e736d" instance_seqno: 1756821973100173) as {username='kudu', principal='kudu/127.1.172.129@KRBTEST.COM'} at 127.1.172.129:51877; Asking this server to re-register.
14:06:26.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:26.486305 2614 master_runner.cc:186] Error getting master registration for 127.1.172.189:35607: Network error: Client connection negotiation failed: client connection to 127.1.172.189:35607: connect: Connection refused (error 111),
14:06:26.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:26.561419 2209 heartbeater.cc:461] Registering TS with master...
14:06:26.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:26.561744 2614 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:06:26.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:26.565346 2692 ts_manager.cc:194] Registered new tserver with Master: 260dd313222a4c7ab6ac8af8728e736d (127.1.172.129:36281)
14:06:26.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:26.724323 2358 heartbeater.cc:344] Connected to a master server at 127.1.172.190:46329
14:06:26.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:26.727586 2692 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "f61769ec2dbe465995cd5b0d0bbb69e6" instance_seqno: 1756821976247767) as {username='kudu', principal='kudu/127.1.172.130@KRBTEST.COM'} at 127.1.172.130:43709; Asking this server to re-register.
14:06:26.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:26.730918 2358 heartbeater.cc:461] Registering TS with master...
14:06:26.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:26.735137 2692 ts_manager.cc:194] Registered new tserver with Master: f61769ec2dbe465995cd5b0d0bbb69e6 (127.1.172.130:42843)
14:06:26.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:26.753755 2729 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:06:26.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:26.754571 2729 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:06:26.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:26.755764 2729 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:06:26.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:26.814129 2729 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
14:06:26.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:26.814496 2729 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:06:26.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:26.814965 2729 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:06:26.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:26.815331 2729 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:06:26.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:26.815667 2729 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:06:26.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:26.815984 2729 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:06:26.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:26.816222 2729 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37243/jwks.json
14:06:26.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:26.883136 2729 master_runner.cc:386] Master server non-default flags:
14:06:26.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:06:26.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.172.148:42901
14:06:26.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:06:26.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:06:26.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/master-1/data
14:06:26.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/master-1/wal
14:06:26.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
14:06:26.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:06:26.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.172.190:46329,127.1.172.189:35607,127.1.172.188:36309
14:06:26.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/krb5kdc/kudu.keytab
14:06:26.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.172.189
14:06:26.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:06:26.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:06:26.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:06:26.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.172.189:35607
14:06:26.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:06:26.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:06:26.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:06:26.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37243/jwks.json
14:06:26.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:06:26.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:06:26.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:06:26.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:06:26.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:06:26.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/master-1/data/info.pb
14:06:26.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.172.189
14:06:26.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33923
14:06:26.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:06:26.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:06:26.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:06:26.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:06:26.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:06:26.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/master-1/logs
14:06:26.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:06:26.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:06:26.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:06:26.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:06:26.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:06:26.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 06466b25e294b4d7d455e6c56263318dc34a80ef
14:06:26.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:06:26.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Sep 2025 13:43:21 UTC on 5fd53c4cbb9d
14:06:26.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7865
14:06:26.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:06:26.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:26.884943 2729 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:06:26.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:26.886325 2729 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:06:26.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:26.910251 2756 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:06:27.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:27.000289 2764 raft_consensus.cc:491] T 00000000000000000000000000000000 P 0f503058e6a84256ab9186070c48408b [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:06:27.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:27.010190 2764 raft_consensus.cc:513] T 00000000000000000000000000000000 P 0f503058e6a84256ab9186070c48408b [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0f05cb838d9d41ee838751c0b5839531" member_type: VOTER last_known_addr { host: "127.1.172.190" port: 46329 } } peers { permanent_uuid: "4cb9c6adfa244ffd8ca9ad5d2c742023" member_type: VOTER last_known_addr { host: "127.1.172.189" port: 35607 } } peers { permanent_uuid: "0f503058e6a84256ab9186070c48408b" member_type: VOTER last_known_addr { host: "127.1.172.188" port: 36309 } }
14:06:27.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:27.082569 2764 leader_election.cc:290] T 00000000000000000000000000000000 P 0f503058e6a84256ab9186070c48408b [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 0f05cb838d9d41ee838751c0b5839531 (127.1.172.190:46329), 4cb9c6adfa244ffd8ca9ad5d2c742023 (127.1.172.189:35607)
14:06:27.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:27.106570 2505 heartbeater.cc:344] Connected to a master server at 127.1.172.190:46329
14:06:27.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:27.107117 2505 heartbeater.cc:499] Master 127.1.172.190:46329 was elected leader, sending a full tablet report...
14:06:27.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:27.114114 2702 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "0f503058e6a84256ab9186070c48408b" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "0f05cb838d9d41ee838751c0b5839531" is_pre_election: true
14:06:27.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:27.115116 2702 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 0f05cb838d9d41ee838751c0b5839531 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 0f503058e6a84256ab9186070c48408b in term 1.
14:06:27.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:27.119474 2564 leader_election.cc:304] T 00000000000000000000000000000000 P 0f503058e6a84256ab9186070c48408b [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 0f05cb838d9d41ee838751c0b5839531, 0f503058e6a84256ab9186070c48408b; no voters:
14:06:27.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:27.125617 2764 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 0f503058e6a84256ab9186070c48408b [term 1 FOLLOWER]: Leader pre-election won for term 2
14:06:27.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:27.134303 2692 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "fd93195deda54438b4dced1460052d02" instance_seqno: 1756821979496130) as {username='kudu', principal='kudu/127.1.172.131@KRBTEST.COM'} at 127.1.172.131:47321; Asking this server to re-register.
14:06:27.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:27.138778 2505 heartbeater.cc:461] Registering TS with master...
14:06:27.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:27.146590 2692 ts_manager.cc:194] Registered new tserver with Master: fd93195deda54438b4dced1460052d02 (127.1.172.131:36021)
14:06:27.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:27.126981 2764 raft_consensus.cc:491] T 00000000000000000000000000000000 P 0f503058e6a84256ab9186070c48408b [term 1 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
14:06:27.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:27.154493 2764 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 0f503058e6a84256ab9186070c48408b [term 1 FOLLOWER]: Advancing to term 2
14:06:27.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:27.153388 2562 leader_election.cc:336] T 00000000000000000000000000000000 P 0f503058e6a84256ab9186070c48408b [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 4cb9c6adfa244ffd8ca9ad5d2c742023 (127.1.172.189:35607): Network error: Client connection negotiation failed: client connection to 127.1.172.189:35607: connect: Connection refused (error 111)
14:06:27.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:27.198072 2764 raft_consensus.cc:513] T 00000000000000000000000000000000 P 0f503058e6a84256ab9186070c48408b [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0f05cb838d9d41ee838751c0b5839531" member_type: VOTER last_known_addr { host: "127.1.172.190" port: 46329 } } peers { permanent_uuid: "4cb9c6adfa244ffd8ca9ad5d2c742023" member_type: VOTER last_known_addr { host: "127.1.172.189" port: 35607 } } peers { permanent_uuid: "0f503058e6a84256ab9186070c48408b" member_type: VOTER last_known_addr { host: "127.1.172.188" port: 36309 } }
14:06:27.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:27.201579 2702 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "0f503058e6a84256ab9186070c48408b" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "0f05cb838d9d41ee838751c0b5839531"
14:06:27.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:27.202193 2702 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 0f05cb838d9d41ee838751c0b5839531 [term 1 FOLLOWER]: Advancing to term 2
14:06:27.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:27.207482 2562 leader_election.cc:336] T 00000000000000000000000000000000 P 0f503058e6a84256ab9186070c48408b [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 4cb9c6adfa244ffd8ca9ad5d2c742023 (127.1.172.189:35607): Network error: Client connection negotiation failed: client connection to 127.1.172.189:35607: connect: Connection refused (error 111)
14:06:27.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:27.203414 2764 leader_election.cc:290] T 00000000000000000000000000000000 P 0f503058e6a84256ab9186070c48408b [CANDIDATE]: Term 2 election: Requested vote from peers 0f05cb838d9d41ee838751c0b5839531 (127.1.172.190:46329), 4cb9c6adfa244ffd8ca9ad5d2c742023 (127.1.172.189:35607)
14:06:27.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:27.221539 2702 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 0f05cb838d9d41ee838751c0b5839531 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 0f503058e6a84256ab9186070c48408b in term 2.
14:06:27.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:27.223496 2564 leader_election.cc:304] T 00000000000000000000000000000000 P 0f503058e6a84256ab9186070c48408b [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 0f05cb838d9d41ee838751c0b5839531, 0f503058e6a84256ab9186070c48408b; no voters: 4cb9c6adfa244ffd8ca9ad5d2c742023
14:06:27.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:27.224690 2764 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 0f503058e6a84256ab9186070c48408b [term 2 FOLLOWER]: Leader election won for term 2
14:06:27.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:27.242066 2764 raft_consensus.cc:695] T 00000000000000000000000000000000 P 0f503058e6a84256ab9186070c48408b [term 2 LEADER]: Becoming Leader. State: Replica: 0f503058e6a84256ab9186070c48408b, State: Running, Role: LEADER
14:06:27.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:27.243999 2764 consensus_queue.cc:237] T 00000000000000000000000000000000 P 0f503058e6a84256ab9186070c48408b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0f05cb838d9d41ee838751c0b5839531" member_type: VOTER last_known_addr { host: "127.1.172.190" port: 46329 } } peers { permanent_uuid: "4cb9c6adfa244ffd8ca9ad5d2c742023" member_type: VOTER last_known_addr { host: "127.1.172.189" port: 35607 } } peers { permanent_uuid: "0f503058e6a84256ab9186070c48408b" member_type: VOTER last_known_addr { host: "127.1.172.188" port: 36309 } }
14:06:27.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:27.275825 2770 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0f503058e6a84256ab9186070c48408b [sys.catalog]: SysCatalogTable state changed. Reason: New leader 0f503058e6a84256ab9186070c48408b. Latest consensus state: current_term: 2 leader_uuid: "0f503058e6a84256ab9186070c48408b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0f05cb838d9d41ee838751c0b5839531" member_type: VOTER last_known_addr { host: "127.1.172.190" port: 46329 } } peers { permanent_uuid: "4cb9c6adfa244ffd8ca9ad5d2c742023" member_type: VOTER last_known_addr { host: "127.1.172.189" port: 35607 } } peers { permanent_uuid: "0f503058e6a84256ab9186070c48408b" member_type: VOTER last_known_addr { host: "127.1.172.188" port: 36309 } } }
14:06:27.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:27.276743 2770 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0f503058e6a84256ab9186070c48408b [sys.catalog]: This master's current role is: LEADER
14:06:27.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:27.290998 2772 catalog_manager.cc:1477] Loading table and tablet metadata into memory...
14:06:27.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:27.316795 2772 catalog_manager.cc:1486] Initializing Kudu cluster ID...
14:06:27.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:27.334255 2772 catalog_manager.cc:1261] Loaded cluster ID: 0c29294b4d6c4be5afc6c01b4bfa41bd
14:06:27.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:27.336407 2772 catalog_manager.cc:1497] Initializing Kudu internal certificate authority...
14:06:27.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:27.346901 2772 catalog_manager.cc:1506] Loading token signing keys...
14:06:27.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:27.362479 2772 catalog_manager.cc:5966] T 00000000000000000000000000000000 P 0f503058e6a84256ab9186070c48408b: Loaded TSK: 0
14:06:27.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:27.365298 2772 catalog_manager.cc:1516] Initializing in-progress tserver states...
14:06:27.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:27.650843 2562 consensus_peers.cc:489] T 00000000000000000000000000000000 P 0f503058e6a84256ab9186070c48408b -> Peer 4cb9c6adfa244ffd8ca9ad5d2c742023 (127.1.172.189:35607): Couldn't send request to peer 4cb9c6adfa244ffd8ca9ad5d2c742023. Status: Network error: Client connection negotiation failed: client connection to 127.1.172.189:35607: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
14:06:27.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:27.842710 2702 raft_consensus.cc:1273] T 00000000000000000000000000000000 P 0f05cb838d9d41ee838751c0b5839531 [term 2 FOLLOWER]: Refusing update from remote peer 0f503058e6a84256ab9186070c48408b: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 2 index: 5. (index mismatch)
14:06:27.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:27.846763 2770 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 0f503058e6a84256ab9186070c48408b [LEADER]: Connected to new peer: Peer: permanent_uuid: "0f05cb838d9d41ee838751c0b5839531" member_type: VOTER last_known_addr { host: "127.1.172.190" port: 46329 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.001s
14:06:27.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:27.953025 2776 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0f05cb838d9d41ee838751c0b5839531 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 0f503058e6a84256ab9186070c48408b. Latest consensus state: current_term: 2 leader_uuid: "0f503058e6a84256ab9186070c48408b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0f05cb838d9d41ee838751c0b5839531" member_type: VOTER last_known_addr { host: "127.1.172.190" port: 46329 } } peers { permanent_uuid: "4cb9c6adfa244ffd8ca9ad5d2c742023" member_type: VOTER last_known_addr { host: "127.1.172.189" port: 35607 } } peers { permanent_uuid: "0f503058e6a84256ab9186070c48408b" member_type: VOTER last_known_addr { host: "127.1.172.188" port: 36309 } } }
14:06:27.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:27.953840 2776 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0f05cb838d9d41ee838751c0b5839531 [sys.catalog]: This master's current role is: FOLLOWER
14:06:27.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:27.958449 2770 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0f503058e6a84256ab9186070c48408b [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "0f503058e6a84256ab9186070c48408b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0f05cb838d9d41ee838751c0b5839531" member_type: VOTER last_known_addr { host: "127.1.172.190" port: 46329 } } peers { permanent_uuid: "4cb9c6adfa244ffd8ca9ad5d2c742023" member_type: VOTER last_known_addr { host: "127.1.172.189" port: 35607 } } peers { permanent_uuid: "0f503058e6a84256ab9186070c48408b" member_type: VOTER last_known_addr { host: "127.1.172.188" port: 36309 } } }
14:06:27.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:27.959790 2770 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0f503058e6a84256ab9186070c48408b [sys.catalog]: This master's current role is: LEADER
14:06:27.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:27.973721 2776 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0f05cb838d9d41ee838751c0b5839531 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "0f503058e6a84256ab9186070c48408b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0f05cb838d9d41ee838751c0b5839531" member_type: VOTER last_known_addr { host: "127.1.172.190" port: 46329 } } peers { permanent_uuid: "4cb9c6adfa244ffd8ca9ad5d2c742023" member_type: VOTER last_known_addr { host: "127.1.172.189" port: 35607 } } peers { permanent_uuid: "0f503058e6a84256ab9186070c48408b" member_type: VOTER last_known_addr { host: "127.1.172.188" port: 36309 } } }
14:06:27.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:27.975703 2776 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0f05cb838d9d41ee838751c0b5839531 [sys.catalog]: This master's current role is: FOLLOWER
14:06:28.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:28.553530 2204 heartbeater.cc:499] Master 127.1.172.188:36309 was elected leader, sending a full tablet report...
14:06:28.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:28.639300 2517 master_runner.cc:186] Error getting master registration for 127.1.172.189:35607: Network error: Client connection negotiation failed: client connection to 127.1.172.189:35607: connect: Connection refused (error 111),
14:06:28.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:28.639842 2517 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:06:28.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:28.680531 2356 heartbeater.cc:499] Master 127.1.172.188:36309 was elected leader, sending a full tablet report...
14:06:28.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:26.928658 2758 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:06:28.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:26.923233 2755 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:06:28.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:28.757907 2757 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
14:06:28.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:28.758507 2729 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:06:28.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:28.763185 2729 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:06:28.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:28.766497 2729 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:06:28.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:28.767956 2729 hybrid_clock.cc:648] HybridClock initialized: now 1756821988767880 us; error 87 us; skew 500 ppm
14:06:28.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:28.768822 2729 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:06:28.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 14:06:28 dist-test-slave-m0vh krb5kdc[1726](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756821988, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.172.189@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:06:28.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:28.775382 2729 init.cc:377] Logged in from keytab as kudu/127.1.172.189@KRBTEST.COM (short username kudu)
14:06:28.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:28.783021 2729 webserver.cc:480] Webserver started at http://127.1.172.189:33923/ using document root <none> and password file <none>
14:06:28.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:28.784138 2729 fs_manager.cc:362] Metadata directory not provided
14:06:28.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:28.784413 2729 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:06:28.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:28.793632 2729 fs_manager.cc:714] Time spent opening directory manager: real 0.006s user 0.002s sys 0.003s
14:06:28.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:28.799115 2790 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:06:28.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:28.800571 2729 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.003s sys 0.000s
14:06:28.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:28.800956 2729 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/master-1/data,/tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/master-1/wal
14:06:28.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4cb9c6adfa244ffd8ca9ad5d2c742023"
14:06:28.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 14:06:04 on dist-test-slave-m0vh"
14:06:28.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:28.803071 2729 fs_report.cc:389] FS layout report
14:06:28.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:06:28.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/master-1/wal
14:06:28.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/master-1/wal
14:06:28.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/master-1/data/data
14:06:28.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:06:28.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:06:28.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:06:28.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:06:28.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:06:28.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:06:28.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:06:28.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:06:28.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:06:28.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:06:28.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:06:28.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:28.878307 2729 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:06:28.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:28.884367 2729 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:06:28.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:28.884887 2729 kserver.cc:163] Server-wide thread pool size limit: 3276
14:06:28.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:28.949445 2503 heartbeater.cc:499] Master 127.1.172.188:36309 was elected leader, sending a full tablet report...
14:06:28.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:28.995517 2729 rpc_server.cc:307] RPC server started. Bound to: 127.1.172.189:35607
14:06:28.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:28.995536 2841 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.1.172.189:35607 every 8 connection(s)
14:06:28.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:28.999545 2729 server_base.cc:1179] Dumped server information to /tmp/dist-test-task50Kl3p/test-tmp/mini-kudu-cluster8154058364016513742/master-1/data/info.pb
14:06:29.002 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:06:29.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:29.001634 1714 external_mini_cluster.cc:1428] Started /tmp/dist-test-task50Kl3p/build/tsan/bin/kudu as pid 2729
14:06:29.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:29.021288 2842 sys_catalog.cc:263] Verifying existing consensus state
14:06:29.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:29.039124 2842 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4cb9c6adfa244ffd8ca9ad5d2c742023: Bootstrap starting.
14:06:29.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:29.065274 2846 negotiation.cc:337] Failed RPC negotiation. Trace:
14:06:29.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 14:06:29.027719 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.1.172.131:37999 (local address 127.1.172.189:35607)
14:06:29.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 14:06:29.048562 (+ 20843us) server_negotiation.cc:206] Beginning negotiation
14:06:29.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 14:06:29.048608 (+ 46us) server_negotiation.cc:399] Waiting for connection header
14:06:29.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 14:06:29.048716 (+ 108us) server_negotiation.cc:407] Connection header received
14:06:29.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 14:06:29.058891 (+ 10175us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
14:06:29.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 14:06:29.058952 (+ 61us) server_negotiation.cc:461] Received NEGOTIATE request from client
14:06:29.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 14:06:29.059602 (+ 650us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
14:06:29.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 14:06:29.060143 (+ 541us) server_negotiation.cc:226] Negotiated authn=SASL
14:06:29.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 14:06:29.063121 (+ 2978us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.1.172.131:37999: BlockingRecv error: recv got EOF from 127.1.172.131:37999 (error 108)
14:06:29.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":19906,"thread_start_us":19470,"threads_started":1}
14:06:29.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 14:06:29 dist-test-slave-m0vh krb5kdc[1726](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756821989, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.172.131@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:06:29.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:29.094834 2385 init.cc:295] Successfully reacquired a new kerberos TGT
14:06:29.097 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.1.172.188:36309(127.1.172.188:36309)] connecting to peer
14:06:29.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:29.104089 2842 log.cc:826] T 00000000000000000000000000000000 P 4cb9c6adfa244ffd8ca9ad5d2c742023: Log is configured to *not* fsync() on all Append() calls
14:06:29.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:29.138904 2842 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4cb9c6adfa244ffd8ca9ad5d2c742023: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
14:06:29.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:29.140223 2842 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4cb9c6adfa244ffd8ca9ad5d2c742023: Bootstrap complete.
14:06:29.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:29.173877 2842 raft_consensus.cc:357] T 00000000000000000000000000000000 P 4cb9c6adfa244ffd8ca9ad5d2c742023 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0f05cb838d9d41ee838751c0b5839531" member_type: VOTER last_known_addr { host: "127.1.172.190" port: 46329 } } peers { permanent_uuid: "4cb9c6adfa244ffd8ca9ad5d2c742023" member_type: VOTER last_known_addr { host: "127.1.172.189" port: 35607 } } peers { permanent_uuid: "0f503058e6a84256ab9186070c48408b" member_type: VOTER last_known_addr { host: "127.1.172.188" port: 36309 } }
14:06:29.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:29.177410 2842 raft_consensus.cc:738] T 00000000000000000000000000000000 P 4cb9c6adfa244ffd8ca9ad5d2c742023 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4cb9c6adfa244ffd8ca9ad5d2c742023, State: Initialized, Role: FOLLOWER
14:06:29.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:29.178578 2842 consensus_queue.cc:260] T 00000000000000000000000000000000 P 4cb9c6adfa244ffd8ca9ad5d2c742023 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0f05cb838d9d41ee838751c0b5839531" member_type: VOTER last_known_addr { host: "127.1.172.190" port: 46329 } } peers { permanent_uuid: "4cb9c6adfa244ffd8ca9ad5d2c742023" member_type: VOTER last_known_addr { host: "127.1.172.189" port: 35607 } } peers { permanent_uuid: "0f503058e6a84256ab9186070c48408b" member_type: VOTER last_known_addr { host: "127.1.172.188" port: 36309 } }
14:06:29.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:29.191051 2842 sys_catalog.cc:564] T 00000000000000000000000000000000 P 4cb9c6adfa244ffd8ca9ad5d2c742023 [sys.catalog]: configured and running, proceeding with master startup.
14:06:29.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 14:06:29 dist-test-slave-m0vh krb5kdc[1726](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756821983, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.172.188@KRBTEST.COM for kudu/127.1.172.189@KRBTEST.COM
14:06:29.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:29.206182 2850 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4cb9c6adfa244ffd8ca9ad5d2c742023 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0f05cb838d9d41ee838751c0b5839531" member_type: VOTER last_known_addr { host: "127.1.172.190" port: 46329 } } peers { permanent_uuid: "4cb9c6adfa244ffd8ca9ad5d2c742023" member_type: VOTER last_known_addr { host: "127.1.172.189" port: 35607 } } peers { permanent_uuid: "0f503058e6a84256ab9186070c48408b" member_type: VOTER last_known_addr { host: "127.1.172.188" port: 36309 } } }
14:06:29.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:29.207122 2850 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4cb9c6adfa244ffd8ca9ad5d2c742023 [sys.catalog]: This master's current role is: FOLLOWER
14:06:29.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:29.255967 2817 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 4cb9c6adfa244ffd8ca9ad5d2c742023 [term 1 FOLLOWER]: Advancing to term 2
14:06:29.356 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.1.172.190:46329(127.1.172.190:46329)] connecting to peer
14:06:29.359 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.1.172.189:35607(127.1.172.189:35607)] connecting to peer
14:06:29.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:29.359967 2862 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4cb9c6adfa244ffd8ca9ad5d2c742023 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "0f503058e6a84256ab9186070c48408b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0f05cb838d9d41ee838751c0b5839531" member_type: VOTER last_known_addr { host: "127.1.172.190" port: 46329 } } peers { permanent_uuid: "4cb9c6adfa244ffd8ca9ad5d2c742023" member_type: VOTER last_known_addr { host: "127.1.172.189" port: 35607 } } peers { permanent_uuid: "0f503058e6a84256ab9186070c48408b" member_type: VOTER last_known_addr { host: "127.1.172.188" port: 36309 } } }
14:06:29.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:29.363688 2862 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4cb9c6adfa244ffd8ca9ad5d2c742023 [sys.catalog]: This master's current role is: FOLLOWER
14:06:29.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:29.367870 2850 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4cb9c6adfa244ffd8ca9ad5d2c742023 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 0f503058e6a84256ab9186070c48408b. Latest consensus state: current_term: 2 leader_uuid: "0f503058e6a84256ab9186070c48408b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0f05cb838d9d41ee838751c0b5839531" member_type: VOTER last_known_addr { host: "127.1.172.190" port: 46329 } } peers { permanent_uuid: "4cb9c6adfa244ffd8ca9ad5d2c742023" member_type: VOTER last_known_addr { host: "127.1.172.189" port: 35607 } } peers { permanent_uuid: "0f503058e6a84256ab9186070c48408b" member_type: VOTER last_known_addr { host: "127.1.172.188" port: 36309 } } }
14:06:29.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:29.376838 2850 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4cb9c6adfa244ffd8ca9ad5d2c742023 [sys.catalog]: This master's current role is: FOLLOWER
14:06:29.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:29.399683 2866 catalog_manager.cc:1261] Loaded cluster ID: 0c29294b4d6c4be5afc6c01b4bfa41bd
14:06:29.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:29.402837 2866 catalog_manager.cc:1554] T 00000000000000000000000000000000 P 4cb9c6adfa244ffd8ca9ad5d2c742023: loading cluster ID for follower catalog manager: success
14:06:29.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:29.418200 2866 catalog_manager.cc:1576] T 00000000000000000000000000000000 P 4cb9c6adfa244ffd8ca9ad5d2c742023: acquiring CA information for follower catalog manager: success
14:06:29.426 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.1.172.190:46329(127.1.172.190:46329)] Successfully connected to peer
14:06:29.435 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.1.172.188:36309(127.1.172.188:36309)] Successfully connected to peer
14:06:29.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:29.453450 2866 catalog_manager.cc:1604] T 00000000000000000000000000000000 P 4cb9c6adfa244ffd8ca9ad5d2c742023: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
14:06:29.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 14:06:29 dist-test-slave-m0vh krb5kdc[1726](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756821988, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.172.189@KRBTEST.COM for kudu/127.1.172.190@KRBTEST.COM
14:06:29.579 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.1.172.189:35607(127.1.172.189:35607)] Successfully connected to peer
14:06:29.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:29.585109 2203 heartbeater.cc:344] Connected to a master server at 127.1.172.189:35607
14:06:29.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:29.594604 2807 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "260dd313222a4c7ab6ac8af8728e736d" instance_seqno: 1756821973100173) as {username='kudu', principal='kudu/127.1.172.129@KRBTEST.COM'} at 127.1.172.129:54477; Asking this server to re-register.
14:06:29.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:29.596926 2203 heartbeater.cc:461] Registering TS with master...
14:06:29.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:29.603037 2807 ts_manager.cc:194] Registered new tserver with Master: 260dd313222a4c7ab6ac8af8728e736d (127.1.172.129:36281)
14:06:29.610 [WARN - kudu-nio-1] (SecurityContext.java:206) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
14:06:29.615 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.172.188:36309(127.1.172.188:36309)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
14:06:29.615 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.172.190:46329(127.1.172.190:46329)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
14:06:29.617 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.172.188:36309: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
14:06:29.617 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.172.190:46329: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
14:06:29.619 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.1.172.190:46329(127.1.172.190:46329)] handling channelInactive
14:06:29.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:29.620353 2871 negotiation.cc:337] Failed RPC negotiation. Trace:
14:06:29.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 14:06:29.426127 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:49702 (local address 127.1.172.190:46329)
14:06:29.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 14:06:29.450254 (+ 24127us) server_negotiation.cc:206] Beginning negotiation
14:06:29.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 14:06:29.450305 (+ 51us) server_negotiation.cc:399] Waiting for connection header
14:06:29.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 14:06:29.531756 (+ 81451us) server_negotiation.cc:407] Connection header received
14:06:29.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 14:06:29.583236 (+ 51480us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
14:06:29.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 14:06:29.583304 (+ 68us) server_negotiation.cc:461] Received NEGOTIATE request from client
14:06:29.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 14:06:29.583673 (+ 369us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
14:06:29.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 14:06:29.583921 (+ 248us) server_negotiation.cc:226] Negotiated authn=SASL
14:06:29.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 14:06:29.618776 (+ 34855us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:49702: BlockingRecv error: recv got EOF from 127.0.0.1:49702 (error 108)
14:06:29.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":22783,"thread_start_us":21966,"threads_started":1}
14:06:29.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:29.620968 2870 negotiation.cc:337] Failed RPC negotiation. Trace:
14:06:29.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 14:06:29.419007 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:42144 (local address 127.1.172.188:36309)
14:06:29.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 14:06:29.444725 (+ 25718us) server_negotiation.cc:206] Beginning negotiation
14:06:29.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 14:06:29.444810 (+ 85us) server_negotiation.cc:399] Waiting for connection header
14:06:29.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 14:06:29.531795 (+ 86985us) server_negotiation.cc:407] Connection header received
14:06:29.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 14:06:29.578953 (+ 47158us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
14:06:29.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 14:06:29.578995 (+ 42us) server_negotiation.cc:461] Received NEGOTIATE request from client
14:06:29.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 14:06:29.579382 (+ 387us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
14:06:29.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 14:06:29.579632 (+ 250us) server_negotiation.cc:226] Negotiated authn=SASL
14:06:29.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 14:06:29.619490 (+ 39858us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:42144: BlockingRecv error: recv got EOF from 127.0.0.1:42144 (error 108)
14:06:29.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":24349,"thread_start_us":27670,"threads_started":1}
14:06:29.623 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.172.189:35607(127.1.172.189:35607)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
14:06:29.624 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.172.189:35607: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
14:06:29.629 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1523) Notify RPC KuduRpc(method=ListTabletServers, tablet=null, attempt=1, TimeoutTracker(timeout=30000, elapsed=620), Traces: [0ms] refreshing cache from master, [58ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.172.188:36309, [334ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.172.190:46329, [337ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.172.189:35607, [593ms] Sub RPC ConnectToMaster: received response from server master-127.1.172.188:36309: Not authorized: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, [594ms] Sub RPC ConnectToMaster: received response from server master-127.1.172.190:46329: Not authorized: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, [601ms] Sub RPC ConnectToMaster: received response from server master-127.1.172.189:35607: Not authorized: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, deferred=null) after lookup exception
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.1.172.188:36309,127.1.172.190:46329,127.1.172.189:35607). Exceptions received: [org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268) [main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) [main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387) [main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376) [main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) [async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) [async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315) [async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286) [async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) [async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) [async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) [main/:?]
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331) [main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256) [main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) [main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) [main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) [main/:?]
at org.apache.kudu.client.Connection.cleanup(Connection.java:737) [main/:?]
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455) [main/:?]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) [netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) [netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) [netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) [netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) [netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_275]
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.1.172.188:36309,127.1.172.190:46329,127.1.172.189:35607). Exceptions received: [org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
14:06:29.635 [DEBUG - kudu-nio-0] (KuduRpc.java:260) Handling a callback on RPC KuduRpc(method=ListTabletServers, tablet=null, attempt=1, TimeoutTracker(timeout=30000, elapsed=627), Traces: [0ms] refreshing cache from master, [58ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.172.188:36309, [334ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.172.190:46329, [337ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.172.189:35607, [593ms] Sub RPC ConnectToMaster: received response from server master-127.1.172.188:36309: Not authorized: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, [594ms] Sub RPC ConnectToMaster: received response from server master-127.1.172.190:46329: Not authorized: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, [601ms] Sub RPC ConnectToMaster: received response from server master-127.1.172.189:35607: Not authorized: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, deferred=null) with no deferred attached!
14:06:29.636 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.1.172.188:36309(127.1.172.188:36309)] handling channelInactive
14:06:29.637 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.1.172.189:35607(127.1.172.189:35607)] handling channelInactive
14:06:29.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:29.638427 2846 negotiation.cc:337] Failed RPC negotiation. Trace:
14:06:29.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 14:06:29.426234 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:53020 (local address 127.1.172.189:35607)
14:06:29.638 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.1.172.188:36309(127.1.172.188:36309)] disconnecting while in state TERMINATED
14:06:29.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 14:06:29.427590 (+ 1356us) server_negotiation.cc:206] Beginning negotiation
14:06:29.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 14:06:29.427654 (+ 64us) server_negotiation.cc:399] Waiting for connection header
14:06:29.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 14:06:29.579620 (+151966us) server_negotiation.cc:407] Connection header received
14:06:29.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 14:06:29.580543 (+ 923us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
14:06:29.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 14:06:29.580574 (+ 31us) server_negotiation.cc:461] Received NEGOTIATE request from client
14:06:29.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 14:06:29.580813 (+ 239us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
14:06:29.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 14:06:29.580973 (+ 160us) server_negotiation.cc:226] Negotiated authn=SASL
14:06:29.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 14:06:29.636754 (+ 55781us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:53020: BlockingRecv error: recv got EOF from 127.0.0.1:53020 (error 108)
14:06:29.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":184}
14:06:29.640 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.1.172.190:46329(127.1.172.190:46329)] disconnecting while in state TERMINATED
14:06:29.640 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.1.172.189:35607(127.1.172.189:35607)] disconnecting while in state TERMINATED
14:06:29.647 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2704) Releasing all remaining resources
14:06:29.650 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
14:06:29.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:29.660823 1714 external_mini_cluster.cc:1658] Killing /tmp/dist-test-task50Kl3p/build/tsan/bin/kudu with pid 2040
14:06:29.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:29.692484 1714 external_mini_cluster.cc:1658] Killing /tmp/dist-test-task50Kl3p/build/tsan/bin/kudu with pid 2218
14:06:29.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:29.727172 1714 external_mini_cluster.cc:1658] Killing /tmp/dist-test-task50Kl3p/build/tsan/bin/kudu with pid 2369
14:06:29.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:29.766916 2770 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0f503058e6a84256ab9186070c48408b [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "0f503058e6a84256ab9186070c48408b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0f05cb838d9d41ee838751c0b5839531" member_type: VOTER last_known_addr { host: "127.1.172.190" port: 46329 } } peers { permanent_uuid: "4cb9c6adfa244ffd8ca9ad5d2c742023" member_type: VOTER last_known_addr { host: "127.1.172.189" port: 35607 } } peers { permanent_uuid: "0f503058e6a84256ab9186070c48408b" member_type: VOTER last_known_addr { host: "127.1.172.188" port: 36309 } } }
14:06:29.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:29.768118 1714 external_mini_cluster.cc:1658] Killing /tmp/dist-test-task50Kl3p/build/tsan/bin/kudu with pid 2614
14:06:29.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:29.768081 2770 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0f503058e6a84256ab9186070c48408b [sys.catalog]: This master's current role is: LEADER
14:06:29.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:29.809659 2564 connection.cc:537] client connection to 127.1.172.190:46329 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
14:06:29.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:29.810302 1714 external_mini_cluster.cc:1658] Killing /tmp/dist-test-task50Kl3p/build/tsan/bin/kudu with pid 2729
14:06:29.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:29.818099 2564 consensus_peers.cc:489] T 00000000000000000000000000000000 P 0f503058e6a84256ab9186070c48408b -> Peer 0f05cb838d9d41ee838751c0b5839531 (127.1.172.190:46329): Couldn't send request to peer 0f05cb838d9d41ee838751c0b5839531. Status: Network error: Client connection negotiation failed: client connection to 127.1.172.190:46329: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
14:06:29.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:29.839915 1714 external_mini_cluster.cc:1658] Killing /tmp/dist-test-task50Kl3p/build/tsan/bin/kudu with pid 2517
14:06:31.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-09-02T14:06:31Z chronyd exiting
14:06:31.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:31.464077 1559 debug-util.cc:398] Leaking SignalData structure 0x7b08000b7280 after lost signal to thread 1434
14:06:31.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 14:06:31.465363 1559 debug-util.cc:398] Leaking SignalData structure 0x7b08000b72a0 after lost signal to thread 1562
14:06:32.119 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
14:06:32.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:32.120733 987 external_mini_cluster.cc:1658] Killing /tmp/dist-test-task50Kl3p/build/tsan/bin/kudu with pid 1270
14:06:32.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:32.148667 987 external_mini_cluster.cc:1658] Killing /tmp/dist-test-task50Kl3p/build/tsan/bin/kudu with pid 1430
14:06:32.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:32.185660 987 external_mini_cluster.cc:1658] Killing /tmp/dist-test-task50Kl3p/build/tsan/bin/kudu with pid 1570
14:06:32.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:32.213518 987 external_mini_cluster.cc:1658] Killing /tmp/dist-test-task50Kl3p/build/tsan/bin/kudu with pid 999
14:06:32.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:32.246050 987 external_mini_cluster.cc:1658] Killing /tmp/dist-test-task50Kl3p/build/tsan/bin/kudu with pid 1069
14:06:32.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 14:06:32.293231 987 external_mini_cluster.cc:1658] Killing /tmp/dist-test-task50Kl3p/build/tsan/bin/kudu with pid 1144
14:06:33.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-09-02T14:06:33Z chronyd exiting
14:06:34.481 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.1.172.188:36309,127.1.172.190:46329,127.1.172.189:35607). Exceptions received: [org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available]
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.cleanup(Connection.java:737) ~[main/:?]
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455) ~[main/:?]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.1.172.188:36309,127.1.172.190:46329,127.1.172.189:35607). Exceptions received: [org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available]
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:748)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more