14:09:43.868 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
14:09:43.868 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskBD8el9/test-tmp
14:09:43.868 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
14:09:43.868 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskBD8el9/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
14:09:43.869 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4741578727841542250"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
14:09:44.194 [DEBUG - main] (MiniKuduCluster.java:195) Response:
14:09:44.194 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
14:09:44.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:44.197435 20583 mini_oidc.cc:150] Starting JWKS server
14:09:44.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:44.201083 20583 webserver.cc:480] Webserver started at http://127.0.0.1:43835/ using document root <none> and password file <none>
14:09:44.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:44.201514 20583 mini_oidc.cc:186] Starting OIDC Discovery server
14:09:44.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:44.203167 20583 webserver.cc:480] Webserver started at http://127.0.0.1:39219/ using document root <none> and password file <none>
14:09:44.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-08-30T14:09:44Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
14:09:44.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-08-30T14:09:44Z Disabled control of system clock
14:09:44.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:44.265481 20583 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskBD8el9/build/tsan/bin/kudu
14:09:44.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskBD8el9/build/tsan/bin/kudu
14:09:44.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4741578727841542250/master-0/wal
14:09:44.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4741578727841542250/master-0/data
14:09:44.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:09:44.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:09:44.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:09:44.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:09:44.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:09:44.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:09:44.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4741578727841542250/master-0/logs
14:09:44.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4741578727841542250/master-0/data/info.pb
14:09:44.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:09:44.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:09:44.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:09:44.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:09:44.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:09:44.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:09:44.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:09:44.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:09:44.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:09:44.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:09:44.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:09:44.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:09:44.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.25.254:39857
14:09:44.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.25.254
14:09:44.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:09:44.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.25.212:34167
14:09:44.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:09:44.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:09:44.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:09:44.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:09:44.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.25.254:39857,127.20.25.253:41261,127.20.25.252:39473
14:09:44.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:09:44.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43835/jwks.json with env {}
14:09:44.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:09:44.565764 20595 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:09:44.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:09:44.566318 20595 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:09:44.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:09:44.566787 20595 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:09:44.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:09:44.597975 20595 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:09:44.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:09:44.598271 20595 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:09:44.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:09:44.598472 20595 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:09:44.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:09:44.598668 20595 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:09:44.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:09:44.598855 20595 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:09:44.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:09:44.599005 20595 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:43835/jwks.json
14:09:44.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:44.634788 20595 master_runner.cc:386] Master server non-default flags:
14:09:44.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:09:44.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.25.212:34167
14:09:44.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:09:44.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:09:44.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4741578727841542250/master-0/data
14:09:44.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4741578727841542250/master-0/wal
14:09:44.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:09:44.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.25.254:39857,127.20.25.253:41261,127.20.25.252:39473
14:09:44.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:09:44.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:09:44.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:09:44.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.25.254:39857
14:09:44.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:09:44.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:09:44.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:09:44.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43835/jwks.json
14:09:44.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:09:44.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:09:44.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4741578727841542250/master-0/data/info.pb
14:09:44.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.25.254
14:09:44.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:09:44.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:09:44.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:09:44.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:09:44.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:09:44.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:09:44.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4741578727841542250/master-0/logs
14:09:44.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:09:44.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:09:44.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:09:44.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:09:44.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:09:44.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 06466b25e294b4d7d455e6c56263318dc34a80ef
14:09:44.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:09:44.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 30 Aug 2025 13:43:24 UTC on 5fd53c4cbb9d
14:09:44.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7817
14:09:44.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:09:44.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:44.636152 20595 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:09:44.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:44.637751 20595 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:09:44.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:09:44.649250 20601 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:09:45.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:09:44.649796 20602 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:09:45.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:09:45.847206 20603 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
14:09:45.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:09:45.848606 20604 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:09:45.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:45.848641 20595 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:09:45.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:45.852142 20595 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:09:45.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:45.855214 20595 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:09:45.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:45.856673 20595 hybrid_clock.cc:648] HybridClock initialized: now 1756562985856624 us; error 57 us; skew 500 ppm
14:09:45.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:45.857589 20595 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:09:45.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:45.864432 20595 webserver.cc:480] Webserver started at http://127.20.25.254:37983/ using document root <none> and password file <none>
14:09:45.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:45.865367 20595 fs_manager.cc:362] Metadata directory not provided
14:09:45.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:45.865557 20595 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:09:45.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:45.865958 20595 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
14:09:45.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:45.871009 20595 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4741578727841542250/master-0/data/instance:
14:09:45.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cdefc3252423421091089cde650d6710"
14:09:45.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-30 14:09:45 on dist-test-slave-xkqm"
14:09:45.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:45.872116 20595 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4741578727841542250/master-0/wal/instance:
14:09:45.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cdefc3252423421091089cde650d6710"
14:09:45.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-30 14:09:45 on dist-test-slave-xkqm"
14:09:45.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:45.879703 20595 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.002s sys 0.004s
14:09:45.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:45.885509 20611 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:09:45.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:45.886641 20595 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
14:09:45.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:45.886940 20595 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4741578727841542250/master-0/data,/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4741578727841542250/master-0/wal
14:09:45.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cdefc3252423421091089cde650d6710"
14:09:45.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-30 14:09:45 on dist-test-slave-xkqm"
14:09:45.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:45.887245 20595 fs_report.cc:389] FS layout report
14:09:45.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:09:45.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4741578727841542250/master-0/wal
14:09:45.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4741578727841542250/master-0/wal
14:09:45.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4741578727841542250/master-0/data/data
14:09:45.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:09:45.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:09:45.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:09:45.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:09:45.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:09:45.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:09:45.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:09:45.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:09:45.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:09:45.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:09:45.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:09:45.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:45.936455 20595 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:09:45.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:45.937871 20595 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:09:45.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:45.938330 20595 kserver.cc:163] Server-wide thread pool size limit: 3276
14:09:46.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:46.010452 20595 rpc_server.cc:307] RPC server started. Bound to: 127.20.25.254:39857
14:09:46.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:46.010517 20662 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.20.25.254:39857 every 8 connection(s)
14:09:46.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:46.013096 20595 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4741578727841542250/master-0/data/info.pb
14:09:46.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:46.017989 20663 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
14:09:46.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:46.023530 20583 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskBD8el9/build/tsan/bin/kudu as pid 20595
14:09:46.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:46.024082 20583 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4741578727841542250/master-0/wal/instance
14:09:46.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:46.029551 20583 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskBD8el9/build/tsan/bin/kudu
14:09:46.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskBD8el9/build/tsan/bin/kudu
14:09:46.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4741578727841542250/master-1/wal
14:09:46.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4741578727841542250/master-1/data
14:09:46.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:09:46.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:09:46.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:09:46.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:09:46.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:09:46.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:09:46.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4741578727841542250/master-1/logs
14:09:46.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4741578727841542250/master-1/data/info.pb
14:09:46.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:09:46.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:09:46.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:09:46.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:09:46.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:09:46.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:09:46.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:09:46.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:09:46.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:09:46.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:09:46.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:09:46.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:09:46.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.25.253:41261
14:09:46.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.25.253
14:09:46.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:09:46.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.25.212:34167
14:09:46.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:09:46.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:09:46.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:09:46.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:09:46.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.25.254:39857,127.20.25.253:41261,127.20.25.252:39473
14:09:46.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:09:46.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43835/jwks.json with env {}
14:09:46.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:46.032601 20663 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.25.254" port: 39857 } has no permanent_uuid. Determining permanent_uuid...
14:09:46.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:46.067481 20663 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.25.253" port: 41261 } has no permanent_uuid. Determining permanent_uuid...
14:09:46.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:09:46.069903 20613 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.20.25.253:41261: connect: Connection refused (error 111)
14:09:46.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:09:46.074433 20663 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.20.25.253:41261: Network error: Client connection negotiation failed: client connection to 127.20.25.253:41261: connect: Connection refused (error 111)
14:09:46.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:46.123925 20663 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.25.253" port: 41261 } attempt: 1
14:09:46.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:09:46.128917 20663 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.20.25.253:41261: Network error: Client connection negotiation failed: client connection to 127.20.25.253:41261: connect: Connection refused (error 111)
14:09:46.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:46.197331 20663 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.25.253" port: 41261 } attempt: 2
14:09:46.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:09:46.201705 20663 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.20.25.253:41261: Network error: Client connection negotiation failed: client connection to 127.20.25.253:41261: connect: Connection refused (error 111)
14:09:46.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:46.293143 20663 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.25.253" port: 41261 } attempt: 3
14:09:46.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:09:46.297446 20663 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.20.25.253:41261: Network error: Client connection negotiation failed: client connection to 127.20.25.253:41261: connect: Connection refused (error 111)
14:09:46.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:09:46.342146 20665 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:09:46.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:09:46.342792 20665 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:09:46.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:09:46.343360 20665 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:09:46.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:09:46.374743 20665 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:09:46.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:09:46.375231 20665 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:09:46.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:09:46.375593 20665 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:09:46.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:09:46.375972 20665 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:09:46.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:09:46.376195 20665 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:09:46.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:09:46.376380 20665 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:43835/jwks.json
14:09:46.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:46.411864 20665 master_runner.cc:386] Master server non-default flags:
14:09:46.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:09:46.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.25.212:34167
14:09:46.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:09:46.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:09:46.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4741578727841542250/master-1/data
14:09:46.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4741578727841542250/master-1/wal
14:09:46.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:09:46.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.25.254:39857,127.20.25.253:41261,127.20.25.252:39473
14:09:46.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:09:46.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:09:46.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:09:46.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.25.253:41261
14:09:46.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:09:46.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:09:46.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:09:46.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43835/jwks.json
14:09:46.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:09:46.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:09:46.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4741578727841542250/master-1/data/info.pb
14:09:46.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.25.253
14:09:46.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:09:46.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:09:46.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:09:46.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:09:46.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:09:46.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:09:46.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4741578727841542250/master-1/logs
14:09:46.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:09:46.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:09:46.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:09:46.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:09:46.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:09:46.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 06466b25e294b4d7d455e6c56263318dc34a80ef
14:09:46.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:09:46.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 30 Aug 2025 13:43:24 UTC on 5fd53c4cbb9d
14:09:46.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7817
14:09:46.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:09:46.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:46.413372 20665 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:09:46.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:46.415002 20665 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:09:46.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:09:46.426047 20674 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:09:46.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:46.440860 20663 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.25.253" port: 41261 } attempt: 4
14:09:46.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:09:46.445371 20663 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.20.25.253:41261: Network error: Client connection negotiation failed: client connection to 127.20.25.253:41261: connect: Connection refused (error 111)
14:09:46.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:46.745040 20663 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.25.253" port: 41261 } attempt: 5
14:09:46.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:09:46.749419 20663 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.20.25.253:41261: Network error: Client connection negotiation failed: client connection to 127.20.25.253:41261: connect: Connection refused (error 111)
14:09:47.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:47.297039 20663 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.25.253" port: 41261 } attempt: 6
14:09:47.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:09:47.304247 20663 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.20.25.253:41261: Network error: Client connection negotiation failed: client connection to 127.20.25.253:41261: connect: Connection refused (error 111)
14:09:47.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:09:46.427537 20675 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:09:47.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:09:46.430182 20677 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:09:47.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:09:47.612936 20676 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
14:09:47.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:47.613015 20665 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:09:47.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:47.617074 20665 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:09:47.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:47.620102 20665 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:09:47.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:47.621593 20665 hybrid_clock.cc:648] HybridClock initialized: now 1756562987621549 us; error 58 us; skew 500 ppm
14:09:47.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:47.622437 20665 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:09:47.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:47.628530 20665 webserver.cc:480] Webserver started at http://127.20.25.253:33617/ using document root <none> and password file <none>
14:09:47.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:47.629480 20665 fs_manager.cc:362] Metadata directory not provided
14:09:47.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:47.629698 20665 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:09:47.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:47.630174 20665 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
14:09:47.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:47.634559 20665 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4741578727841542250/master-1/data/instance:
14:09:47.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f136080c8f594bbd8190f0fdb1153a14"
14:09:47.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-30 14:09:47 on dist-test-slave-xkqm"
14:09:47.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:47.635682 20665 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4741578727841542250/master-1/wal/instance:
14:09:47.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f136080c8f594bbd8190f0fdb1153a14"
14:09:47.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-30 14:09:47 on dist-test-slave-xkqm"
14:09:47.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:47.642752 20665 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.005s sys 0.000s
14:09:47.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:47.648488 20686 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:09:47.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:47.649669 20665 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.001s
14:09:47.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:47.650059 20665 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4741578727841542250/master-1/data,/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4741578727841542250/master-1/wal
14:09:47.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f136080c8f594bbd8190f0fdb1153a14"
14:09:47.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-30 14:09:47 on dist-test-slave-xkqm"
14:09:47.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:47.650403 20665 fs_report.cc:389] FS layout report
14:09:47.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:09:47.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4741578727841542250/master-1/wal
14:09:47.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4741578727841542250/master-1/wal
14:09:47.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4741578727841542250/master-1/data/data
14:09:47.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:09:47.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:09:47.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:09:47.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:09:47.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:09:47.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:09:47.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:09:47.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:09:47.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:09:47.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:09:47.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:09:47.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:47.710569 20665 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:09:47.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:47.712069 20665 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:09:47.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:47.712550 20665 kserver.cc:163] Server-wide thread pool size limit: 3276
14:09:47.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:47.785897 20665 rpc_server.cc:307] RPC server started. Bound to: 127.20.25.253:41261
14:09:47.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:47.785984 20737 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.20.25.253:41261 every 8 connection(s)
14:09:47.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:47.788607 20665 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4741578727841542250/master-1/data/info.pb
14:09:47.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:47.793589 20738 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
14:09:47.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:47.798846 20583 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskBD8el9/build/tsan/bin/kudu as pid 20665
14:09:47.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:47.799362 20583 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4741578727841542250/master-1/wal/instance
14:09:47.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:47.802423 20583 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskBD8el9/build/tsan/bin/kudu
14:09:47.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskBD8el9/build/tsan/bin/kudu
14:09:47.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4741578727841542250/master-2/wal
14:09:47.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4741578727841542250/master-2/data
14:09:47.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:09:47.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:09:47.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:09:47.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:09:47.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:09:47.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:09:47.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4741578727841542250/master-2/logs
14:09:47.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4741578727841542250/master-2/data/info.pb
14:09:47.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:09:47.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:09:47.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:09:47.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:09:47.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:09:47.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:09:47.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:09:47.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:09:47.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:09:47.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:09:47.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:09:47.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:09:47.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.25.252:39473
14:09:47.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.25.252
14:09:47.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:09:47.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.25.212:34167
14:09:47.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:09:47.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:09:47.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:09:47.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:09:47.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.25.254:39857,127.20.25.253:41261,127.20.25.252:39473
14:09:47.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:09:47.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43835/jwks.json with env {}
14:09:47.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:47.808758 20738 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.25.254" port: 39857 } has no permanent_uuid. Determining permanent_uuid...
14:09:47.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:47.830161 20738 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.25.253" port: 41261 } has no permanent_uuid. Determining permanent_uuid...
14:09:47.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:47.843662 20738 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.25.252" port: 39473 } has no permanent_uuid. Determining permanent_uuid...
14:09:47.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:09:47.846344 20687 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.20.25.252:39473: connect: Connection refused (error 111)
14:09:47.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:09:47.850852 20738 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.20.25.252:39473: Network error: Client connection negotiation failed: client connection to 127.20.25.252:39473: connect: Connection refused (error 111)
14:09:47.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:47.900398 20738 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.25.252" port: 39473 } attempt: 1
14:09:47.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:09:47.905105 20738 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.20.25.252:39473: Network error: Client connection negotiation failed: client connection to 127.20.25.252:39473: connect: Connection refused (error 111)
14:09:47.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:47.973508 20738 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.25.252" port: 39473 } attempt: 2
14:09:47.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:09:47.977741 20738 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.20.25.252:39473: Network error: Client connection negotiation failed: client connection to 127.20.25.252:39473: connect: Connection refused (error 111)
14:09:48.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:48.069181 20738 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.25.252" port: 39473 } attempt: 3
14:09:48.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:09:48.073282 20738 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.20.25.252:39473: Network error: Client connection negotiation failed: client connection to 127.20.25.252:39473: connect: Connection refused (error 111)
14:09:48.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:09:48.111320 20740 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:09:48.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:09:48.111899 20740 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:09:48.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:09:48.112424 20740 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:09:48.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:09:48.144748 20740 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:09:48.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:09:48.145035 20740 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:09:48.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:09:48.145231 20740 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:09:48.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:09:48.145433 20740 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:09:48.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:09:48.145622 20740 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:09:48.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:09:48.145763 20740 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:43835/jwks.json
14:09:48.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:48.181748 20740 master_runner.cc:386] Master server non-default flags:
14:09:48.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:09:48.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.25.212:34167
14:09:48.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:09:48.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:09:48.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4741578727841542250/master-2/data
14:09:48.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4741578727841542250/master-2/wal
14:09:48.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:09:48.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.25.254:39857,127.20.25.253:41261,127.20.25.252:39473
14:09:48.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:09:48.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:09:48.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:09:48.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.25.252:39473
14:09:48.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:09:48.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:09:48.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:09:48.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43835/jwks.json
14:09:48.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:09:48.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:09:48.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4741578727841542250/master-2/data/info.pb
14:09:48.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.25.252
14:09:48.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:09:48.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:09:48.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:09:48.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:09:48.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:09:48.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:09:48.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4741578727841542250/master-2/logs
14:09:48.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:09:48.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:09:48.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:09:48.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:09:48.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:09:48.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 06466b25e294b4d7d455e6c56263318dc34a80ef
14:09:48.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:09:48.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 30 Aug 2025 13:43:24 UTC on 5fd53c4cbb9d
14:09:48.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7817
14:09:48.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:09:48.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:48.183033 20740 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:09:48.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:48.184741 20740 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:09:48.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:09:48.195268 20750 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:09:48.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:48.216658 20738 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.25.252" port: 39473 } attempt: 4
14:09:48.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:09:48.220374 20738 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.20.25.252:39473: Network error: Client connection negotiation failed: client connection to 127.20.25.252:39473: connect: Connection refused (error 111)
14:09:48.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:48.364912 20663 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.25.253" port: 41261 } attempt: 7
14:09:48.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:48.398665 20663 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.25.252" port: 39473 } has no permanent_uuid. Determining permanent_uuid...
14:09:48.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:09:48.407490 20663 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.20.25.252:39473: Network error: Client connection negotiation failed: client connection to 127.20.25.252:39473: connect: Connection refused (error 111)
14:09:48.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:48.466051 20663 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.25.252" port: 39473 } attempt: 1
14:09:48.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:09:48.472457 20663 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.20.25.252:39473: Network error: Client connection negotiation failed: client connection to 127.20.25.252:39473: connect: Connection refused (error 111)
14:09:48.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:48.519814 20738 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.25.252" port: 39473 } attempt: 5
14:09:48.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:09:48.526693 20738 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.20.25.252:39473: Network error: Client connection negotiation failed: client connection to 127.20.25.252:39473: connect: Connection refused (error 111)
14:09:48.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:48.554046 20663 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.25.252" port: 39473 } attempt: 2
14:09:48.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:09:48.560436 20663 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.20.25.252:39473: Network error: Client connection negotiation failed: client connection to 127.20.25.252:39473: connect: Connection refused (error 111)
14:09:48.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:48.646075 20663 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.25.252" port: 39473 } attempt: 3
14:09:48.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:09:48.652959 20663 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.20.25.252:39473: Network error: Client connection negotiation failed: client connection to 127.20.25.252:39473: connect: Connection refused (error 111)
14:09:48.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:48.793536 20663 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.25.252" port: 39473 } attempt: 4
14:09:48.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:09:48.800951 20663 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.20.25.252:39473: Network error: Client connection negotiation failed: client connection to 127.20.25.252:39473: connect: Connection refused (error 111)
14:09:49.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:49.074357 20738 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.25.252" port: 39473 } attempt: 6
14:09:49.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:49.084540 20663 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.25.252" port: 39473 } attempt: 5
14:09:49.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:09:49.086496 20738 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.20.25.252:39473: Network error: Client connection negotiation failed: client connection to 127.20.25.252:39473: connect: Connection refused (error 111)
14:09:49.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:09:49.092011 20663 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.20.25.252:39473: Network error: Client connection negotiation failed: client connection to 127.20.25.252:39473: connect: Connection refused (error 111)
14:09:49.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:09:48.196669 20751 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:09:49.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:09:49.448848 20753 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:09:49.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:09:49.451119 20752 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1251 milliseconds
14:09:49.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:49.451290 20740 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:09:49.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:49.452576 20740 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:09:49.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:49.455603 20740 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:09:49.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:49.457020 20740 hybrid_clock.cc:648] HybridClock initialized: now 1756562989456978 us; error 54 us; skew 500 ppm
14:09:49.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:49.457768 20740 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:09:49.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:49.464264 20740 webserver.cc:480] Webserver started at http://127.20.25.252:33805/ using document root <none> and password file <none>
14:09:49.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:49.465132 20740 fs_manager.cc:362] Metadata directory not provided
14:09:49.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:49.465329 20740 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:09:49.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:49.465721 20740 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
14:09:49.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:49.470171 20740 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4741578727841542250/master-2/data/instance:
14:09:49.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9a6984bece2f437bbbc7f1fc867adabb"
14:09:49.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-30 14:09:49 on dist-test-slave-xkqm"
14:09:49.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:49.471534 20740 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4741578727841542250/master-2/wal/instance:
14:09:49.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9a6984bece2f437bbbc7f1fc867adabb"
14:09:49.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-30 14:09:49 on dist-test-slave-xkqm"
14:09:49.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:49.478730 20740 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.009s sys 0.000s
14:09:49.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:49.484505 20765 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:09:49.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:49.485585 20740 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
14:09:49.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:49.485958 20740 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4741578727841542250/master-2/data,/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4741578727841542250/master-2/wal
14:09:49.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9a6984bece2f437bbbc7f1fc867adabb"
14:09:49.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-30 14:09:49 on dist-test-slave-xkqm"
14:09:49.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:49.486318 20740 fs_report.cc:389] FS layout report
14:09:49.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:09:49.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4741578727841542250/master-2/wal
14:09:49.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4741578727841542250/master-2/wal
14:09:49.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4741578727841542250/master-2/data/data
14:09:49.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:09:49.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:09:49.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:09:49.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:09:49.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:09:49.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:09:49.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:09:49.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:09:49.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:09:49.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:09:49.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:09:49.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:49.534089 20740 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:09:49.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:49.535619 20740 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:09:49.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:49.536113 20740 kserver.cc:163] Server-wide thread pool size limit: 3276
14:09:49.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:49.607997 20740 rpc_server.cc:307] RPC server started. Bound to: 127.20.25.252:39473
14:09:49.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:49.608080 20816 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.20.25.252:39473 every 8 connection(s)
14:09:49.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:49.610719 20740 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4741578727841542250/master-2/data/info.pb
14:09:49.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:49.613969 20583 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskBD8el9/build/tsan/bin/kudu as pid 20740
14:09:49.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:49.614485 20583 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4741578727841542250/master-2/wal/instance
14:09:49.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:49.616672 20817 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
14:09:49.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:49.633698 20817 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.25.254" port: 39857 } has no permanent_uuid. Determining permanent_uuid...
14:09:49.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:49.646093 20663 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.25.252" port: 39473 } attempt: 6
14:09:49.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:49.667107 20817 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.25.253" port: 41261 } has no permanent_uuid. Determining permanent_uuid...
14:09:49.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:49.679147 20817 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.25.252" port: 39473 } has no permanent_uuid. Determining permanent_uuid...
14:09:49.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:49.681553 20663 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P cdefc3252423421091089cde650d6710: Bootstrap starting.
14:09:49.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:49.688473 20663 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P cdefc3252423421091089cde650d6710: Neither blocks nor log segments found. Creating new log.
14:09:49.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:49.690902 20663 log.cc:826] T 00000000000000000000000000000000 P cdefc3252423421091089cde650d6710: Log is configured to *not* fsync() on all Append() calls
14:09:49.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:49.696364 20663 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P cdefc3252423421091089cde650d6710: No bootstrap required, opened a new log
14:09:49.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:49.700098 20817 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9a6984bece2f437bbbc7f1fc867adabb: Bootstrap starting.
14:09:49.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:49.707252 20817 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 9a6984bece2f437bbbc7f1fc867adabb: Neither blocks nor log segments found. Creating new log.
14:09:49.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:49.709053 20817 log.cc:826] T 00000000000000000000000000000000 P 9a6984bece2f437bbbc7f1fc867adabb: Log is configured to *not* fsync() on all Append() calls
14:09:49.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:49.713900 20817 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9a6984bece2f437bbbc7f1fc867adabb: No bootstrap required, opened a new log
14:09:49.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:49.718196 20663 raft_consensus.cc:357] T 00000000000000000000000000000000 P cdefc3252423421091089cde650d6710 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cdefc3252423421091089cde650d6710" member_type: VOTER last_known_addr { host: "127.20.25.254" port: 39857 } } peers { permanent_uuid: "f136080c8f594bbd8190f0fdb1153a14" member_type: VOTER last_known_addr { host: "127.20.25.253" port: 41261 } } peers { permanent_uuid: "9a6984bece2f437bbbc7f1fc867adabb" member_type: VOTER last_known_addr { host: "127.20.25.252" port: 39473 } }
14:09:49.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:49.719262 20663 raft_consensus.cc:383] T 00000000000000000000000000000000 P cdefc3252423421091089cde650d6710 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:09:49.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:49.719595 20663 raft_consensus.cc:738] T 00000000000000000000000000000000 P cdefc3252423421091089cde650d6710 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: cdefc3252423421091089cde650d6710, State: Initialized, Role: FOLLOWER
14:09:49.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:49.720394 20663 consensus_queue.cc:260] T 00000000000000000000000000000000 P cdefc3252423421091089cde650d6710 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cdefc3252423421091089cde650d6710" member_type: VOTER last_known_addr { host: "127.20.25.254" port: 39857 } } peers { permanent_uuid: "f136080c8f594bbd8190f0fdb1153a14" member_type: VOTER last_known_addr { host: "127.20.25.253" port: 41261 } } peers { permanent_uuid: "9a6984bece2f437bbbc7f1fc867adabb" member_type: VOTER last_known_addr { host: "127.20.25.252" port: 39473 } }
14:09:49.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:49.723133 20829 sys_catalog.cc:455] T 00000000000000000000000000000000 P cdefc3252423421091089cde650d6710 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cdefc3252423421091089cde650d6710" member_type: VOTER last_known_addr { host: "127.20.25.254" port: 39857 } } peers { permanent_uuid: "f136080c8f594bbd8190f0fdb1153a14" member_type: VOTER last_known_addr { host: "127.20.25.253" port: 41261 } } peers { permanent_uuid: "9a6984bece2f437bbbc7f1fc867adabb" member_type: VOTER last_known_addr { host: "127.20.25.252" port: 39473 } } }
14:09:49.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:49.723944 20829 sys_catalog.cc:458] T 00000000000000000000000000000000 P cdefc3252423421091089cde650d6710 [sys.catalog]: This master's current role is: FOLLOWER
14:09:49.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:49.724536 20663 sys_catalog.cc:564] T 00000000000000000000000000000000 P cdefc3252423421091089cde650d6710 [sys.catalog]: configured and running, proceeding with master startup.
14:09:49.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:49.735945 20817 raft_consensus.cc:357] T 00000000000000000000000000000000 P 9a6984bece2f437bbbc7f1fc867adabb [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cdefc3252423421091089cde650d6710" member_type: VOTER last_known_addr { host: "127.20.25.254" port: 39857 } } peers { permanent_uuid: "f136080c8f594bbd8190f0fdb1153a14" member_type: VOTER last_known_addr { host: "127.20.25.253" port: 41261 } } peers { permanent_uuid: "9a6984bece2f437bbbc7f1fc867adabb" member_type: VOTER last_known_addr { host: "127.20.25.252" port: 39473 } }
14:09:49.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:49.736660 20817 raft_consensus.cc:383] T 00000000000000000000000000000000 P 9a6984bece2f437bbbc7f1fc867adabb [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:09:49.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:49.736866 20817 raft_consensus.cc:738] T 00000000000000000000000000000000 P 9a6984bece2f437bbbc7f1fc867adabb [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9a6984bece2f437bbbc7f1fc867adabb, State: Initialized, Role: FOLLOWER
14:09:49.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:49.737572 20817 consensus_queue.cc:260] T 00000000000000000000000000000000 P 9a6984bece2f437bbbc7f1fc867adabb [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cdefc3252423421091089cde650d6710" member_type: VOTER last_known_addr { host: "127.20.25.254" port: 39857 } } peers { permanent_uuid: "f136080c8f594bbd8190f0fdb1153a14" member_type: VOTER last_known_addr { host: "127.20.25.253" port: 41261 } } peers { permanent_uuid: "9a6984bece2f437bbbc7f1fc867adabb" member_type: VOTER last_known_addr { host: "127.20.25.252" port: 39473 } }
14:09:49.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:49.745862 20836 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9a6984bece2f437bbbc7f1fc867adabb [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cdefc3252423421091089cde650d6710" member_type: VOTER last_known_addr { host: "127.20.25.254" port: 39857 } } peers { permanent_uuid: "f136080c8f594bbd8190f0fdb1153a14" member_type: VOTER last_known_addr { host: "127.20.25.253" port: 41261 } } peers { permanent_uuid: "9a6984bece2f437bbbc7f1fc867adabb" member_type: VOTER last_known_addr { host: "127.20.25.252" port: 39473 } } }
14:09:49.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:49.746732 20836 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9a6984bece2f437bbbc7f1fc867adabb [sys.catalog]: This master's current role is: FOLLOWER
14:09:49.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:49.748169 20817 sys_catalog.cc:564] T 00000000000000000000000000000000 P 9a6984bece2f437bbbc7f1fc867adabb [sys.catalog]: configured and running, proceeding with master startup.
14:09:49.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:09:49.767453 20841 catalog_manager.cc:1560] T 00000000000000000000000000000000 P cdefc3252423421091089cde650d6710: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:09:49.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:09:49.767776 20841 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:09:49.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:09:49.791143 20771 tablet.cc:2378] T 00000000000000000000000000000000 P 9a6984bece2f437bbbc7f1fc867adabb: Can't schedule compaction. Clean time has not been advanced past its initial value.
14:09:49.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:09:49.816154 20853 catalog_manager.cc:1560] T 00000000000000000000000000000000 P 9a6984bece2f437bbbc7f1fc867adabb: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:09:49.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:09:49.816550 20853 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:09:49.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:49.830235 20829 raft_consensus.cc:491] T 00000000000000000000000000000000 P cdefc3252423421091089cde650d6710 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:09:49.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:49.830751 20829 raft_consensus.cc:513] T 00000000000000000000000000000000 P cdefc3252423421091089cde650d6710 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cdefc3252423421091089cde650d6710" member_type: VOTER last_known_addr { host: "127.20.25.254" port: 39857 } } peers { permanent_uuid: "f136080c8f594bbd8190f0fdb1153a14" member_type: VOTER last_known_addr { host: "127.20.25.253" port: 41261 } } peers { permanent_uuid: "9a6984bece2f437bbbc7f1fc867adabb" member_type: VOTER last_known_addr { host: "127.20.25.252" port: 39473 } }
14:09:49.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:49.834981 20829 leader_election.cc:290] T 00000000000000000000000000000000 P cdefc3252423421091089cde650d6710 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers f136080c8f594bbd8190f0fdb1153a14 (127.20.25.253:41261), 9a6984bece2f437bbbc7f1fc867adabb (127.20.25.252:39473)
14:09:49.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:49.835090 20792 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "cdefc3252423421091089cde650d6710" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9a6984bece2f437bbbc7f1fc867adabb" is_pre_election: true
14:09:49.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:49.835996 20792 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 9a6984bece2f437bbbc7f1fc867adabb [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate cdefc3252423421091089cde650d6710 in term 0.
14:09:49.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:49.837697 20612 leader_election.cc:304] T 00000000000000000000000000000000 P cdefc3252423421091089cde650d6710 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 9a6984bece2f437bbbc7f1fc867adabb, cdefc3252423421091089cde650d6710; no voters:
14:09:49.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:49.838784 20829 raft_consensus.cc:2802] T 00000000000000000000000000000000 P cdefc3252423421091089cde650d6710 [term 0 FOLLOWER]: Leader pre-election won for term 1
14:09:49.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:49.839135 20829 raft_consensus.cc:491] T 00000000000000000000000000000000 P cdefc3252423421091089cde650d6710 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
14:09:49.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:49.839450 20829 raft_consensus.cc:3058] T 00000000000000000000000000000000 P cdefc3252423421091089cde650d6710 [term 0 FOLLOWER]: Advancing to term 1
14:09:49.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:49.757047 20595 master_runner.cc:186] Error getting master registration for 127.20.25.253:41261: OK, instance_id { permanent_uuid: "f136080c8f594bbd8190f0fdb1153a14" instance_seqno: 1756562987763741 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
14:09:49.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:49.846527 20595 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:09:49.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:49.854229 20829 raft_consensus.cc:513] T 00000000000000000000000000000000 P cdefc3252423421091089cde650d6710 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cdefc3252423421091089cde650d6710" member_type: VOTER last_known_addr { host: "127.20.25.254" port: 39857 } } peers { permanent_uuid: "f136080c8f594bbd8190f0fdb1153a14" member_type: VOTER last_known_addr { host: "127.20.25.253" port: 41261 } } peers { permanent_uuid: "9a6984bece2f437bbbc7f1fc867adabb" member_type: VOTER last_known_addr { host: "127.20.25.252" port: 39473 } }
14:09:49.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:49.867233 20792 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "cdefc3252423421091089cde650d6710" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9a6984bece2f437bbbc7f1fc867adabb"
14:09:49.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:49.867825 20792 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 9a6984bece2f437bbbc7f1fc867adabb [term 0 FOLLOWER]: Advancing to term 1
14:09:49.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:49.802009 20740 master_runner.cc:186] Error getting master registration for 127.20.25.253:41261: OK, instance_id { permanent_uuid: "f136080c8f594bbd8190f0fdb1153a14" instance_seqno: 1756562987763741 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
14:09:49.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:49.877203 20792 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 9a6984bece2f437bbbc7f1fc867adabb [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate cdefc3252423421091089cde650d6710 in term 1.
14:09:49.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:49.881115 20740 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:09:49.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:49.884125 20829 leader_election.cc:290] T 00000000000000000000000000000000 P cdefc3252423421091089cde650d6710 [CANDIDATE]: Term 1 election: Requested vote from peers f136080c8f594bbd8190f0fdb1153a14 (127.20.25.253:41261), 9a6984bece2f437bbbc7f1fc867adabb (127.20.25.252:39473)
14:09:49.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:49.887724 20612 leader_election.cc:304] T 00000000000000000000000000000000 P cdefc3252423421091089cde650d6710 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 9a6984bece2f437bbbc7f1fc867adabb, cdefc3252423421091089cde650d6710; no voters:
14:09:49.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:49.888958 20829 raft_consensus.cc:2802] T 00000000000000000000000000000000 P cdefc3252423421091089cde650d6710 [term 1 FOLLOWER]: Leader election won for term 1
14:09:49.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:49.908761 20829 raft_consensus.cc:695] T 00000000000000000000000000000000 P cdefc3252423421091089cde650d6710 [term 1 LEADER]: Becoming Leader. State: Replica: cdefc3252423421091089cde650d6710, State: Running, Role: LEADER
14:09:49.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:49.910151 20829 consensus_queue.cc:237] T 00000000000000000000000000000000 P cdefc3252423421091089cde650d6710 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cdefc3252423421091089cde650d6710" member_type: VOTER last_known_addr { host: "127.20.25.254" port: 39857 } } peers { permanent_uuid: "f136080c8f594bbd8190f0fdb1153a14" member_type: VOTER last_known_addr { host: "127.20.25.253" port: 41261 } } peers { permanent_uuid: "9a6984bece2f437bbbc7f1fc867adabb" member_type: VOTER last_known_addr { host: "127.20.25.252" port: 39473 } }
14:09:49.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:49.927191 20855 sys_catalog.cc:455] T 00000000000000000000000000000000 P cdefc3252423421091089cde650d6710 [sys.catalog]: SysCatalogTable state changed. Reason: New leader cdefc3252423421091089cde650d6710. Latest consensus state: current_term: 1 leader_uuid: "cdefc3252423421091089cde650d6710" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cdefc3252423421091089cde650d6710" member_type: VOTER last_known_addr { host: "127.20.25.254" port: 39857 } } peers { permanent_uuid: "f136080c8f594bbd8190f0fdb1153a14" member_type: VOTER last_known_addr { host: "127.20.25.253" port: 41261 } } peers { permanent_uuid: "9a6984bece2f437bbbc7f1fc867adabb" member_type: VOTER last_known_addr { host: "127.20.25.252" port: 39473 } } }
14:09:49.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:49.928077 20855 sys_catalog.cc:458] T 00000000000000000000000000000000 P cdefc3252423421091089cde650d6710 [sys.catalog]: This master's current role is: LEADER
14:09:49.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:49.938305 20857 catalog_manager.cc:1477] Loading table and tablet metadata into memory...
14:09:49.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:49.945856 20857 catalog_manager.cc:1486] Initializing Kudu cluster ID...
14:09:49.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:09:49.947475 20617 tablet.cc:2378] T 00000000000000000000000000000000 P cdefc3252423421091089cde650d6710: Can't schedule compaction. Clean time has not been advanced past its initial value.
14:09:49.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:49.970628 20792 raft_consensus.cc:1273] T 00000000000000000000000000000000 P 9a6984bece2f437bbbc7f1fc867adabb [term 1 FOLLOWER]: Refusing update from remote peer cdefc3252423421091089cde650d6710: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
14:09:49.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:49.972666 20829 consensus_queue.cc:1035] T 00000000000000000000000000000000 P cdefc3252423421091089cde650d6710 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9a6984bece2f437bbbc7f1fc867adabb" member_type: VOTER last_known_addr { host: "127.20.25.252" port: 39473 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
14:09:50.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:50.007999 20836 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9a6984bece2f437bbbc7f1fc867adabb [sys.catalog]: SysCatalogTable state changed. Reason: New leader cdefc3252423421091089cde650d6710. Latest consensus state: current_term: 1 leader_uuid: "cdefc3252423421091089cde650d6710" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cdefc3252423421091089cde650d6710" member_type: VOTER last_known_addr { host: "127.20.25.254" port: 39857 } } peers { permanent_uuid: "f136080c8f594bbd8190f0fdb1153a14" member_type: VOTER last_known_addr { host: "127.20.25.253" port: 41261 } } peers { permanent_uuid: "9a6984bece2f437bbbc7f1fc867adabb" member_type: VOTER last_known_addr { host: "127.20.25.252" port: 39473 } } }
14:09:50.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:50.008735 20836 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9a6984bece2f437bbbc7f1fc867adabb [sys.catalog]: This master's current role is: FOLLOWER
14:09:50.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:50.019354 20858 mvcc.cc:204] Tried to move back new op lower bound from 7194882006891622400 to 7194882006682558464. Current Snapshot: MvccSnapshot[applied={T|T < 7194882006891622400}]
14:09:50.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:50.025408 20855 sys_catalog.cc:455] T 00000000000000000000000000000000 P cdefc3252423421091089cde650d6710 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "cdefc3252423421091089cde650d6710" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cdefc3252423421091089cde650d6710" member_type: VOTER last_known_addr { host: "127.20.25.254" port: 39857 } } peers { permanent_uuid: "f136080c8f594bbd8190f0fdb1153a14" member_type: VOTER last_known_addr { host: "127.20.25.253" port: 41261 } } peers { permanent_uuid: "9a6984bece2f437bbbc7f1fc867adabb" member_type: VOTER last_known_addr { host: "127.20.25.252" port: 39473 } } }
14:09:50.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:50.026149 20855 sys_catalog.cc:458] T 00000000000000000000000000000000 P cdefc3252423421091089cde650d6710 [sys.catalog]: This master's current role is: LEADER
14:09:50.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:50.027674 20859 mvcc.cc:204] Tried to move back new op lower bound from 7194882006891622400 to 7194882006682558464. Current Snapshot: MvccSnapshot[applied={T|T < 7194882006891622400}]
14:09:50.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:50.031068 20836 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9a6984bece2f437bbbc7f1fc867adabb [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "cdefc3252423421091089cde650d6710" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cdefc3252423421091089cde650d6710" member_type: VOTER last_known_addr { host: "127.20.25.254" port: 39857 } } peers { permanent_uuid: "f136080c8f594bbd8190f0fdb1153a14" member_type: VOTER last_known_addr { host: "127.20.25.253" port: 41261 } } peers { permanent_uuid: "9a6984bece2f437bbbc7f1fc867adabb" member_type: VOTER last_known_addr { host: "127.20.25.252" port: 39473 } } }
14:09:50.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:50.031778 20836 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9a6984bece2f437bbbc7f1fc867adabb [sys.catalog]: This master's current role is: FOLLOWER
14:09:50.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:50.041921 20857 catalog_manager.cc:1349] Generated new cluster ID: e3af747605e04e98a2116d2f0de4f3aa
14:09:50.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:50.042294 20857 catalog_manager.cc:1497] Initializing Kudu internal certificate authority...
14:09:50.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:50.074424 20857 catalog_manager.cc:1372] Generated new certificate authority record
14:09:50.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:50.076589 20857 catalog_manager.cc:1506] Loading token signing keys...
14:09:50.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:50.105257 20857 catalog_manager.cc:5955] T 00000000000000000000000000000000 P cdefc3252423421091089cde650d6710: Generated new TSK 0
14:09:50.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:50.106439 20857 catalog_manager.cc:1516] Initializing in-progress tserver states...
14:09:50.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:50.139173 20583 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskBD8el9/build/tsan/bin/kudu
14:09:50.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskBD8el9/build/tsan/bin/kudu
14:09:50.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4741578727841542250/ts-0/wal
14:09:50.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4741578727841542250/ts-0/data
14:09:50.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:09:50.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:09:50.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:09:50.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:09:50.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:09:50.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:09:50.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4741578727841542250/ts-0/logs
14:09:50.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4741578727841542250/ts-0/data/info.pb
14:09:50.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:09:50.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:09:50.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:09:50.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:09:50.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:09:50.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:09:50.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:09:50.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:09:50.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
14:09:50.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:09:50.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.25.193:0
14:09:50.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.25.193
14:09:50.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.25.193
14:09:50.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:09:50.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.25.254:39857,127.20.25.253:41261,127.20.25.252:39473
14:09:50.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.25.212:34167
14:09:50.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:09:50.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:09:50.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
14:09:50.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:50.147081 20738 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.25.252" port: 39473 } attempt: 7
14:09:50.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:50.197055 20738 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f136080c8f594bbd8190f0fdb1153a14: Bootstrap starting.
14:09:50.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:50.208269 20738 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P f136080c8f594bbd8190f0fdb1153a14: Neither blocks nor log segments found. Creating new log.
14:09:50.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:50.217304 20738 log.cc:826] T 00000000000000000000000000000000 P f136080c8f594bbd8190f0fdb1153a14: Log is configured to *not* fsync() on all Append() calls
14:09:50.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:50.236198 20738 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f136080c8f594bbd8190f0fdb1153a14: No bootstrap required, opened a new log
14:09:50.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:50.263592 20738 raft_consensus.cc:357] T 00000000000000000000000000000000 P f136080c8f594bbd8190f0fdb1153a14 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cdefc3252423421091089cde650d6710" member_type: VOTER last_known_addr { host: "127.20.25.254" port: 39857 } } peers { permanent_uuid: "f136080c8f594bbd8190f0fdb1153a14" member_type: VOTER last_known_addr { host: "127.20.25.253" port: 41261 } } peers { permanent_uuid: "9a6984bece2f437bbbc7f1fc867adabb" member_type: VOTER last_known_addr { host: "127.20.25.252" port: 39473 } }
14:09:50.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:50.264709 20738 raft_consensus.cc:383] T 00000000000000000000000000000000 P f136080c8f594bbd8190f0fdb1153a14 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:09:50.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:50.265197 20738 raft_consensus.cc:738] T 00000000000000000000000000000000 P f136080c8f594bbd8190f0fdb1153a14 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f136080c8f594bbd8190f0fdb1153a14, State: Initialized, Role: FOLLOWER
14:09:50.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:50.266392 20738 consensus_queue.cc:260] T 00000000000000000000000000000000 P f136080c8f594bbd8190f0fdb1153a14 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cdefc3252423421091089cde650d6710" member_type: VOTER last_known_addr { host: "127.20.25.254" port: 39857 } } peers { permanent_uuid: "f136080c8f594bbd8190f0fdb1153a14" member_type: VOTER last_known_addr { host: "127.20.25.253" port: 41261 } } peers { permanent_uuid: "9a6984bece2f437bbbc7f1fc867adabb" member_type: VOTER last_known_addr { host: "127.20.25.252" port: 39473 } }
14:09:50.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:50.289505 20866 sys_catalog.cc:455] T 00000000000000000000000000000000 P f136080c8f594bbd8190f0fdb1153a14 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cdefc3252423421091089cde650d6710" member_type: VOTER last_known_addr { host: "127.20.25.254" port: 39857 } } peers { permanent_uuid: "f136080c8f594bbd8190f0fdb1153a14" member_type: VOTER last_known_addr { host: "127.20.25.253" port: 41261 } } peers { permanent_uuid: "9a6984bece2f437bbbc7f1fc867adabb" member_type: VOTER last_known_addr { host: "127.20.25.252" port: 39473 } } }
14:09:50.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:50.290582 20866 sys_catalog.cc:458] T 00000000000000000000000000000000 P f136080c8f594bbd8190f0fdb1153a14 [sys.catalog]: This master's current role is: FOLLOWER
14:09:50.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:50.295843 20713 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "cdefc3252423421091089cde650d6710" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f136080c8f594bbd8190f0fdb1153a14" is_pre_election: true
14:09:50.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:50.296752 20713 raft_consensus.cc:2466] T 00000000000000000000000000000000 P f136080c8f594bbd8190f0fdb1153a14 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate cdefc3252423421091089cde650d6710 in term 0.
14:09:50.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:50.297749 20712 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "cdefc3252423421091089cde650d6710" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f136080c8f594bbd8190f0fdb1153a14"
14:09:50.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:50.298323 20712 raft_consensus.cc:3058] T 00000000000000000000000000000000 P f136080c8f594bbd8190f0fdb1153a14 [term 0 FOLLOWER]: Advancing to term 1
14:09:50.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:50.306968 20712 raft_consensus.cc:2466] T 00000000000000000000000000000000 P f136080c8f594bbd8190f0fdb1153a14 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate cdefc3252423421091089cde650d6710 in term 1.
14:09:50.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:50.308352 20711 raft_consensus.cc:1273] T 00000000000000000000000000000000 P f136080c8f594bbd8190f0fdb1153a14 [term 1 FOLLOWER]: Refusing update from remote peer cdefc3252423421091089cde650d6710: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
14:09:50.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:50.314803 20829 consensus_queue.cc:1035] T 00000000000000000000000000000000 P cdefc3252423421091089cde650d6710 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f136080c8f594bbd8190f0fdb1153a14" member_type: VOTER last_known_addr { host: "127.20.25.253" port: 41261 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
14:09:50.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:50.325770 20738 sys_catalog.cc:564] T 00000000000000000000000000000000 P f136080c8f594bbd8190f0fdb1153a14 [sys.catalog]: configured and running, proceeding with master startup.
14:09:50.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:50.380776 20866 sys_catalog.cc:455] T 00000000000000000000000000000000 P f136080c8f594bbd8190f0fdb1153a14 [sys.catalog]: SysCatalogTable state changed. Reason: New leader cdefc3252423421091089cde650d6710. Latest consensus state: current_term: 1 leader_uuid: "cdefc3252423421091089cde650d6710" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cdefc3252423421091089cde650d6710" member_type: VOTER last_known_addr { host: "127.20.25.254" port: 39857 } } peers { permanent_uuid: "f136080c8f594bbd8190f0fdb1153a14" member_type: VOTER last_known_addr { host: "127.20.25.253" port: 41261 } } peers { permanent_uuid: "9a6984bece2f437bbbc7f1fc867adabb" member_type: VOTER last_known_addr { host: "127.20.25.252" port: 39473 } } }
14:09:50.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:50.381521 20866 sys_catalog.cc:458] T 00000000000000000000000000000000 P f136080c8f594bbd8190f0fdb1153a14 [sys.catalog]: This master's current role is: FOLLOWER
14:09:50.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:50.381775 20871 sys_catalog.cc:455] T 00000000000000000000000000000000 P f136080c8f594bbd8190f0fdb1153a14 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "cdefc3252423421091089cde650d6710" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cdefc3252423421091089cde650d6710" member_type: VOTER last_known_addr { host: "127.20.25.254" port: 39857 } } peers { permanent_uuid: "f136080c8f594bbd8190f0fdb1153a14" member_type: VOTER last_known_addr { host: "127.20.25.253" port: 41261 } } peers { permanent_uuid: "9a6984bece2f437bbbc7f1fc867adabb" member_type: VOTER last_known_addr { host: "127.20.25.252" port: 39473 } } }
14:09:50.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:50.382764 20871 sys_catalog.cc:458] T 00000000000000000000000000000000 P f136080c8f594bbd8190f0fdb1153a14 [sys.catalog]: This master's current role is: FOLLOWER
14:09:50.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:50.455513 20882 catalog_manager.cc:1261] Loaded cluster ID: e3af747605e04e98a2116d2f0de4f3aa
14:09:50.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:50.456061 20882 catalog_manager.cc:1554] T 00000000000000000000000000000000 P f136080c8f594bbd8190f0fdb1153a14: loading cluster ID for follower catalog manager: success
14:09:50.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:50.463663 20882 catalog_manager.cc:1576] T 00000000000000000000000000000000 P f136080c8f594bbd8190f0fdb1153a14: acquiring CA information for follower catalog manager: success
14:09:50.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:50.469506 20882 catalog_manager.cc:1604] T 00000000000000000000000000000000 P f136080c8f594bbd8190f0fdb1153a14: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
14:09:50.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:09:50.675559 20863 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:09:50.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:09:50.683284 20863 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:09:50.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:09:50.684162 20863 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:09:50.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:09:50.795043 20863 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:09:50.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:09:50.806138 20863 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.20.25.193
14:09:50.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:50.822000 20853 catalog_manager.cc:1261] Loaded cluster ID: e3af747605e04e98a2116d2f0de4f3aa
14:09:50.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:50.822481 20853 catalog_manager.cc:1554] T 00000000000000000000000000000000 P 9a6984bece2f437bbbc7f1fc867adabb: loading cluster ID for follower catalog manager: success
14:09:50.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:50.829572 20853 catalog_manager.cc:1576] T 00000000000000000000000000000000 P 9a6984bece2f437bbbc7f1fc867adabb: acquiring CA information for follower catalog manager: success
14:09:50.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:50.864665 20853 catalog_manager.cc:1604] T 00000000000000000000000000000000 P 9a6984bece2f437bbbc7f1fc867adabb: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
14:09:50.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:50.871372 20863 tablet_server_runner.cc:78] Tablet server non-default flags:
14:09:50.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:09:50.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.25.212:34167
14:09:50.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:09:50.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:09:50.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4741578727841542250/ts-0/data
14:09:50.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4741578727841542250/ts-0/wal
14:09:50.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:09:50.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:09:50.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.25.193:0
14:09:50.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:09:50.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:09:50.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:09:50.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4741578727841542250/ts-0/data/info.pb
14:09:50.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.25.193
14:09:50.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:09:50.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.25.254:39857,127.20.25.253:41261,127.20.25.252:39473
14:09:50.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:09:50.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:09:50.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:09:50.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:09:50.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:09:50.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.25.193
14:09:50.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4741578727841542250/ts-0/logs
14:09:50.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:09:50.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:09:50.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:09:50.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
14:09:50.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:09:50.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 06466b25e294b4d7d455e6c56263318dc34a80ef
14:09:50.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:09:50.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 30 Aug 2025 13:43:24 UTC on 5fd53c4cbb9d
14:09:50.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7817
14:09:50.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:09:50.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:50.873112 20863 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:09:50.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:50.875280 20863 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:09:50.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:09:50.905164 20889 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:09:50.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:50.959906 20892 sys_catalog.cc:455] T 00000000000000000000000000000000 P cdefc3252423421091089cde650d6710 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "cdefc3252423421091089cde650d6710" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cdefc3252423421091089cde650d6710" member_type: VOTER last_known_addr { host: "127.20.25.254" port: 39857 } } peers { permanent_uuid: "f136080c8f594bbd8190f0fdb1153a14" member_type: VOTER last_known_addr { host: "127.20.25.253" port: 41261 } } peers { permanent_uuid: "9a6984bece2f437bbbc7f1fc867adabb" member_type: VOTER last_known_addr { host: "127.20.25.252" port: 39473 } } }
14:09:50.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:50.962807 20892 sys_catalog.cc:458] T 00000000000000000000000000000000 P cdefc3252423421091089cde650d6710 [sys.catalog]: This master's current role is: LEADER
14:09:53.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:09:52.298777 20888 debug-util.cc:398] Leaking SignalData structure 0x7b0800034ea0 after lost signal to thread 20863
14:09:53.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:09:53.133426 20888 kernel_stack_watchdog.cc:198] Thread 20863 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 388ms:
14:09:53.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
14:09:53.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
14:09:53.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
14:09:53.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
14:09:53.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:09:53.137432 20863 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 2.229s user 0.561s sys 0.977s
14:09:53.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:09:53.138094 20863 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 2.230s user 0.561s sys 0.977s
14:09:53.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:09:50.910238 20890 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:09:53.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:53.147341 20863 server_base.cc:1047] running on GCE node
14:09:53.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:09:53.152729 20897 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:09:53.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:53.154435 20863 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:09:53.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:53.166114 20863 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:09:53.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:53.169909 20863 hybrid_clock.cc:648] HybridClock initialized: now 1756562993169859 us; error 62 us; skew 500 ppm
14:09:53.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:53.171186 20863 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:09:53.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:53.204995 20863 webserver.cc:480] Webserver started at http://127.20.25.193:34831/ using document root <none> and password file <none>
14:09:53.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:53.206604 20863 fs_manager.cc:362] Metadata directory not provided
14:09:53.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:53.207038 20863 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:09:53.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:53.207705 20863 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
14:09:53.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:53.214941 20863 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4741578727841542250/ts-0/data/instance:
14:09:53.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f6031176e1814997883e9e1e612a227a"
14:09:53.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-30 14:09:53 on dist-test-slave-xkqm"
14:09:53.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:53.217069 20863 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4741578727841542250/ts-0/wal/instance:
14:09:53.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f6031176e1814997883e9e1e612a227a"
14:09:53.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-30 14:09:53 on dist-test-slave-xkqm"
14:09:53.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:53.238054 20863 fs_manager.cc:696] Time spent creating directory manager: real 0.020s user 0.015s sys 0.002s
14:09:53.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:53.252040 20902 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:09:53.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:53.253435 20863 fs_manager.cc:730] Time spent opening block manager: real 0.011s user 0.006s sys 0.004s
14:09:53.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:53.253854 20863 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4741578727841542250/ts-0/data,/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4741578727841542250/ts-0/wal
14:09:53.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f6031176e1814997883e9e1e612a227a"
14:09:53.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-30 14:09:53 on dist-test-slave-xkqm"
14:09:53.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:53.254292 20863 fs_report.cc:389] FS layout report
14:09:53.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:09:53.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4741578727841542250/ts-0/wal
14:09:53.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4741578727841542250/ts-0/wal
14:09:53.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4741578727841542250/ts-0/data/data
14:09:53.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:09:53.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:09:53.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:09:53.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:09:53.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:09:53.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:09:53.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:09:53.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:09:53.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:09:53.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:09:53.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:09:53.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:53.350193 20863 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:09:53.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:53.352419 20863 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:09:53.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:53.353017 20863 kserver.cc:163] Server-wide thread pool size limit: 3276
14:09:53.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:53.368249 20863 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
14:09:53.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:53.380266 20863 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
14:09:53.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:53.380586 20863 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.001s sys 0.000s
14:09:53.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:53.380904 20863 ts_tablet_manager.cc:610] Registered 0 tablets
14:09:53.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:53.381127 20863 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
14:09:53.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:53.744068 20863 rpc_server.cc:307] RPC server started. Bound to: 127.20.25.193:43619
14:09:53.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:53.744105 21014 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.20.25.193:43619 every 8 connection(s)
14:09:53.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:53.747725 20863 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4741578727841542250/ts-0/data/info.pb
14:09:53.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:53.754380 20583 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskBD8el9/build/tsan/bin/kudu as pid 20863
14:09:53.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:53.754894 20583 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4741578727841542250/ts-0/wal/instance
14:09:53.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:53.780061 20583 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskBD8el9/build/tsan/bin/kudu
14:09:53.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskBD8el9/build/tsan/bin/kudu
14:09:53.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4741578727841542250/ts-1/wal
14:09:53.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4741578727841542250/ts-1/data
14:09:53.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:09:53.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:09:53.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:09:53.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:09:53.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:09:53.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:09:53.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4741578727841542250/ts-1/logs
14:09:53.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4741578727841542250/ts-1/data/info.pb
14:09:53.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:09:53.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:09:53.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:09:53.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:09:53.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:09:53.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:09:53.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:09:53.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:09:53.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
14:09:53.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:09:53.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.25.194:0
14:09:53.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.25.194
14:09:53.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.25.194
14:09:53.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:09:53.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.25.254:39857,127.20.25.253:41261,127.20.25.252:39473
14:09:53.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.25.212:34167
14:09:53.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:09:53.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:09:53.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
14:09:53.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:53.846704 21016 heartbeater.cc:344] Connected to a master server at 127.20.25.253:41261
14:09:53.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:53.847344 21016 heartbeater.cc:461] Registering TS with master...
14:09:53.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:53.848816 21016 heartbeater.cc:507] Master 127.20.25.253:41261 requested a full tablet report, sending...
14:09:53.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:53.853236 20703 ts_manager.cc:194] Registered new tserver with Master: f6031176e1814997883e9e1e612a227a (127.20.25.193:43619)
14:09:53.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:53.859370 21017 heartbeater.cc:344] Connected to a master server at 127.20.25.254:39857
14:09:53.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:53.862661 21017 heartbeater.cc:461] Registering TS with master...
14:09:53.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:53.863530 21017 heartbeater.cc:507] Master 127.20.25.254:39857 requested a full tablet report, sending...
14:09:53.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:53.861163 21015 heartbeater.cc:344] Connected to a master server at 127.20.25.252:39473
14:09:53.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:53.864398 21015 heartbeater.cc:461] Registering TS with master...
14:09:53.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:53.865315 21015 heartbeater.cc:507] Master 127.20.25.252:39473 requested a full tablet report, sending...
14:09:53.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:53.866253 20628 ts_manager.cc:194] Registered new tserver with Master: f6031176e1814997883e9e1e612a227a (127.20.25.193:43619)
14:09:53.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:53.869140 20628 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.20.25.193:36919
14:09:53.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:53.874277 20782 ts_manager.cc:194] Registered new tserver with Master: f6031176e1814997883e9e1e612a227a (127.20.25.193:43619)
14:09:54.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:09:54.166318 21025 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:09:54.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:09:54.166873 21025 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:09:54.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:09:54.167423 21025 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:09:54.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:09:54.199504 21025 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:09:54.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:09:54.200471 21025 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.20.25.194
14:09:54.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:54.237006 21025 tablet_server_runner.cc:78] Tablet server non-default flags:
14:09:54.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:09:54.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.25.212:34167
14:09:54.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:09:54.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:09:54.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4741578727841542250/ts-1/data
14:09:54.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4741578727841542250/ts-1/wal
14:09:54.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:09:54.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:09:54.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.25.194:0
14:09:54.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:09:54.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:09:54.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:09:54.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4741578727841542250/ts-1/data/info.pb
14:09:54.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.25.194
14:09:54.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:09:54.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.25.254:39857,127.20.25.253:41261,127.20.25.252:39473
14:09:54.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:09:54.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:09:54.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:09:54.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:09:54.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:09:54.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.25.194
14:09:54.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4741578727841542250/ts-1/logs
14:09:54.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:09:54.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:09:54.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:09:54.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
14:09:54.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:09:54.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 06466b25e294b4d7d455e6c56263318dc34a80ef
14:09:54.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:09:54.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 30 Aug 2025 13:43:24 UTC on 5fd53c4cbb9d
14:09:54.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7817
14:09:54.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:09:54.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:54.238667 21025 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:09:54.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:54.240545 21025 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:09:54.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:09:54.253716 21032 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:09:54.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:54.873893 21017 heartbeater.cc:499] Master 127.20.25.254:39857 was elected leader, sending a full tablet report...
14:09:56.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:09:55.657366 21031 debug-util.cc:398] Leaking SignalData structure 0x7b0800034ea0 after lost signal to thread 21025
14:09:56.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:09:56.038206 21031 kernel_stack_watchdog.cc:198] Thread 21025 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 401ms:
14:09:56.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
14:09:56.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
14:09:56.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
14:09:56.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
14:09:56.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:09:56.038682 21025 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.785s user 0.663s sys 1.103s
14:09:56.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:09:56.039167 21025 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.786s user 0.663s sys 1.104s
14:09:56.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:09:54.254063 21033 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:09:56.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:09:56.042924 21035 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:09:56.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:09:56.045570 21034 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection timed out after 1791 milliseconds
14:09:56.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:56.045603 21025 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:09:56.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:56.047039 21025 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:09:56.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:56.049180 21025 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:09:56.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:56.050732 21025 hybrid_clock.cc:648] HybridClock initialized: now 1756562996050689 us; error 48 us; skew 500 ppm
14:09:56.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:56.051642 21025 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:09:56.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:56.057875 21025 webserver.cc:480] Webserver started at http://127.20.25.194:33879/ using document root <none> and password file <none>
14:09:56.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:56.058902 21025 fs_manager.cc:362] Metadata directory not provided
14:09:56.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:56.059147 21025 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:09:56.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:56.059615 21025 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
14:09:56.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:56.064038 21025 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4741578727841542250/ts-1/data/instance:
14:09:56.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e6aa0ffccd1d43b3aa5e0c2f85a5ac8e"
14:09:56.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-30 14:09:56 on dist-test-slave-xkqm"
14:09:56.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:56.065490 21025 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4741578727841542250/ts-1/wal/instance:
14:09:56.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e6aa0ffccd1d43b3aa5e0c2f85a5ac8e"
14:09:56.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-30 14:09:56 on dist-test-slave-xkqm"
14:09:56.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:56.073019 21025 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.006s sys 0.002s
14:09:56.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:56.079010 21045 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:09:56.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:56.080142 21025 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.005s sys 0.001s
14:09:56.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:56.080601 21025 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4741578727841542250/ts-1/data,/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4741578727841542250/ts-1/wal
14:09:56.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e6aa0ffccd1d43b3aa5e0c2f85a5ac8e"
14:09:56.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-30 14:09:56 on dist-test-slave-xkqm"
14:09:56.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:56.080940 21025 fs_report.cc:389] FS layout report
14:09:56.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:09:56.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4741578727841542250/ts-1/wal
14:09:56.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4741578727841542250/ts-1/wal
14:09:56.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4741578727841542250/ts-1/data/data
14:09:56.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:09:56.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:09:56.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:09:56.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:09:56.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:09:56.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:09:56.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:09:56.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:09:56.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:09:56.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:09:56.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:09:56.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:56.136126 21025 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:09:56.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:56.137722 21025 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:09:56.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:56.138199 21025 kserver.cc:163] Server-wide thread pool size limit: 3276
14:09:56.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:56.140816 21025 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
14:09:56.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:56.145293 21025 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
14:09:56.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:56.145524 21025 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
14:09:56.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:56.145830 21025 ts_tablet_manager.cc:610] Registered 0 tablets
14:09:56.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:56.145999 21025 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
14:09:56.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:56.286077 21025 rpc_server.cc:307] RPC server started. Bound to: 127.20.25.194:45547
14:09:56.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:56.286219 21157 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.20.25.194:45547 every 8 connection(s)
14:09:56.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:56.288687 21025 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4741578727841542250/ts-1/data/info.pb
14:09:56.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:56.299181 20583 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskBD8el9/build/tsan/bin/kudu as pid 21025
14:09:56.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:56.299996 20583 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4741578727841542250/ts-1/wal/instance
14:09:56.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:56.312853 20583 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskBD8el9/build/tsan/bin/kudu
14:09:56.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskBD8el9/build/tsan/bin/kudu
14:09:56.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4741578727841542250/ts-2/wal
14:09:56.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4741578727841542250/ts-2/data
14:09:56.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:09:56.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:09:56.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:09:56.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:09:56.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:09:56.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:09:56.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4741578727841542250/ts-2/logs
14:09:56.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4741578727841542250/ts-2/data/info.pb
14:09:56.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:09:56.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:09:56.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:09:56.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:09:56.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:09:56.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:09:56.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:09:56.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:09:56.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
14:09:56.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:09:56.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.25.195:0
14:09:56.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.25.195
14:09:56.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.25.195
14:09:56.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:09:56.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.25.254:39857,127.20.25.253:41261,127.20.25.252:39473
14:09:56.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.25.212:34167
14:09:56.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:09:56.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:09:56.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
14:09:56.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:56.351557 21160 heartbeater.cc:344] Connected to a master server at 127.20.25.254:39857
14:09:56.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:56.352095 21160 heartbeater.cc:461] Registering TS with master...
14:09:56.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:56.354826 21160 heartbeater.cc:507] Master 127.20.25.254:39857 requested a full tablet report, sending...
14:09:56.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:56.356269 21158 heartbeater.cc:344] Connected to a master server at 127.20.25.252:39473
14:09:56.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:56.356645 21158 heartbeater.cc:461] Registering TS with master...
14:09:56.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:56.357558 21158 heartbeater.cc:507] Master 127.20.25.252:39473 requested a full tablet report, sending...
14:09:56.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:56.358830 20628 ts_manager.cc:194] Registered new tserver with Master: e6aa0ffccd1d43b3aa5e0c2f85a5ac8e (127.20.25.194:45547)
14:09:56.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:56.359365 21159 heartbeater.cc:344] Connected to a master server at 127.20.25.253:41261
14:09:56.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:56.359756 21159 heartbeater.cc:461] Registering TS with master...
14:09:56.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:56.360412 20628 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.20.25.194:41971
14:09:56.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:56.360787 21159 heartbeater.cc:507] Master 127.20.25.253:41261 requested a full tablet report, sending...
14:09:56.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:56.362867 20782 ts_manager.cc:194] Registered new tserver with Master: e6aa0ffccd1d43b3aa5e0c2f85a5ac8e (127.20.25.194:45547)
14:09:56.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:56.364480 20703 ts_manager.cc:194] Registered new tserver with Master: e6aa0ffccd1d43b3aa5e0c2f85a5ac8e (127.20.25.194:45547)
14:09:56.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:09:56.664141 21169 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:09:56.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:09:56.664693 21169 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:09:56.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:09:56.665160 21169 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:09:56.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:09:56.697144 21169 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:09:56.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:09:56.697952 21169 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.20.25.195
14:09:56.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:56.734086 21169 tablet_server_runner.cc:78] Tablet server non-default flags:
14:09:56.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:09:56.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.25.212:34167
14:09:56.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:09:56.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:09:56.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4741578727841542250/ts-2/data
14:09:56.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4741578727841542250/ts-2/wal
14:09:56.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:09:56.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:09:56.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.25.195:0
14:09:56.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:09:56.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:09:56.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:09:56.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4741578727841542250/ts-2/data/info.pb
14:09:56.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.25.195
14:09:56.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:09:56.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.25.254:39857,127.20.25.253:41261,127.20.25.252:39473
14:09:56.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:09:56.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:09:56.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:09:56.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:09:56.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:09:56.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.25.195
14:09:56.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4741578727841542250/ts-2/logs
14:09:56.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:09:56.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:09:56.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:09:56.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
14:09:56.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:09:56.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 06466b25e294b4d7d455e6c56263318dc34a80ef
14:09:56.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:09:56.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 30 Aug 2025 13:43:24 UTC on 5fd53c4cbb9d
14:09:56.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7817
14:09:56.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:09:56.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:56.735356 21169 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:09:56.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:56.737032 21169 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:09:56.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:09:56.750608 21175 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:09:57.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:57.364442 21160 heartbeater.cc:499] Master 127.20.25.254:39857 was elected leader, sending a full tablet report...
14:09:58.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:09:56.751827 21176 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:09:58.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:09:58.153970 21174 debug-util.cc:398] Leaking SignalData structure 0x7b0800034ea0 after lost signal to thread 21169
14:09:58.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:09:58.492616 21169 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.742s user 0.580s sys 1.152s
14:09:58.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:09:58.493441 21169 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.743s user 0.580s sys 1.152s
14:09:58.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:09:58.493724 21177 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1741 milliseconds
14:09:58.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:09:58.495260 21178 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:09:58.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:58.495239 21169 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:09:58.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:58.499287 21169 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:09:58.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:58.501611 21169 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:09:58.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:58.503013 21169 hybrid_clock.cc:648] HybridClock initialized: now 1756562998502956 us; error 71 us; skew 500 ppm
14:09:58.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:58.503911 21169 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:09:58.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:58.510334 21169 webserver.cc:480] Webserver started at http://127.20.25.195:37375/ using document root <none> and password file <none>
14:09:58.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:58.511364 21169 fs_manager.cc:362] Metadata directory not provided
14:09:58.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:58.511605 21169 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:09:58.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:58.512068 21169 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
14:09:58.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:58.516685 21169 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4741578727841542250/ts-2/data/instance:
14:09:58.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a47108977fa84e57be5fd20062afeed7"
14:09:58.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-30 14:09:58 on dist-test-slave-xkqm"
14:09:58.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:58.518018 21169 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4741578727841542250/ts-2/wal/instance:
14:09:58.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a47108977fa84e57be5fd20062afeed7"
14:09:58.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-30 14:09:58 on dist-test-slave-xkqm"
14:09:58.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:58.525982 21169 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.002s sys 0.006s
14:09:58.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:58.532315 21186 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:09:58.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:58.533581 21169 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.001s
14:09:58.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:58.533977 21169 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4741578727841542250/ts-2/data,/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4741578727841542250/ts-2/wal
14:09:58.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a47108977fa84e57be5fd20062afeed7"
14:09:58.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-30 14:09:58 on dist-test-slave-xkqm"
14:09:58.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:58.534339 21169 fs_report.cc:389] FS layout report
14:09:58.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:09:58.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4741578727841542250/ts-2/wal
14:09:58.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4741578727841542250/ts-2/wal
14:09:58.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4741578727841542250/ts-2/data/data
14:09:58.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:09:58.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:09:58.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:09:58.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:09:58.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:09:58.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:09:58.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:09:58.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:09:58.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:09:58.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:09:58.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:09:58.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:58.589505 21169 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:09:58.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:58.591045 21169 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:09:58.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:58.591590 21169 kserver.cc:163] Server-wide thread pool size limit: 3276
14:09:58.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:58.594252 21169 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
14:09:58.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:58.598832 21169 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
14:09:58.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:58.599079 21169 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
14:09:58.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:58.599421 21169 ts_tablet_manager.cc:610] Registered 0 tablets
14:09:58.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:58.599596 21169 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
14:09:58.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:58.751013 21169 rpc_server.cc:307] RPC server started. Bound to: 127.20.25.195:38731
14:09:58.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:58.751077 21298 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.20.25.195:38731 every 8 connection(s)
14:09:58.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:58.754689 21169 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4741578727841542250/ts-2/data/info.pb
14:09:58.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:58.757911 20583 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskBD8el9/build/tsan/bin/kudu as pid 21169
14:09:58.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:58.758411 20583 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4741578727841542250/ts-2/wal/instance
14:09:58.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:58.806533 21299 heartbeater.cc:344] Connected to a master server at 127.20.25.252:39473
14:09:58.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:58.807035 21299 heartbeater.cc:461] Registering TS with master...
14:09:58.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:58.808318 21299 heartbeater.cc:507] Master 127.20.25.252:39473 requested a full tablet report, sending...
14:09:58.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:58.808679 21300 heartbeater.cc:344] Connected to a master server at 127.20.25.253:41261
14:09:58.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:58.809031 21300 heartbeater.cc:461] Registering TS with master...
14:09:58.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:58.809226 21303 heartbeater.cc:344] Connected to a master server at 127.20.25.254:39857
14:09:58.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:58.809772 21303 heartbeater.cc:461] Registering TS with master...
14:09:58.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:58.809949 21300 heartbeater.cc:507] Master 127.20.25.253:41261 requested a full tablet report, sending...
14:09:58.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:58.810689 21303 heartbeater.cc:507] Master 127.20.25.254:39857 requested a full tablet report, sending...
14:09:58.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:58.811563 20782 ts_manager.cc:194] Registered new tserver with Master: a47108977fa84e57be5fd20062afeed7 (127.20.25.195:38731)
14:09:58.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:58.812606 20703 ts_manager.cc:194] Registered new tserver with Master: a47108977fa84e57be5fd20062afeed7 (127.20.25.195:38731)
14:09:58.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:58.812896 20628 ts_manager.cc:194] Registered new tserver with Master: a47108977fa84e57be5fd20062afeed7 (127.20.25.195:38731)
14:09:58.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:58.814400 20628 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.20.25.195:34721
14:09:58.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:58.829125 20583 external_mini_cluster.cc:949] 3 TS(s) registered with all masters
14:09:58.843 [DEBUG - main] (MiniKuduCluster.java:195) Response:
14:09:58.843 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
14:09:58.844 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.20.25.254"
port: 39857
}
bound_http_address {
host: "127.20.25.254"
port: 37983
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.20.25.253"
port: 41261
}
bound_http_address {
host: "127.20.25.253"
port: 33617
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.20.25.252"
port: 39473
}
bound_http_address {
host: "127.20.25.252"
port: 33805
}
}
}
14:09:58.844 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
14:09:58.845 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.20.25.193"
port: 43619
}
bound_http_address {
host: "127.20.25.193"
port: 34831
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.20.25.194"
port: 45547
}
bound_http_address {
host: "127.20.25.194"
port: 33879
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.20.25.195"
port: 38731
}
bound_http_address {
host: "127.20.25.195"
port: 37375
}
}
}
14:09:58.845 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
14:09:58.846 [DEBUG - main] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4210330887469000091/krb5kdc/krb5cc
14:09:58.847 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
14:09:58.847 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskBD8el9/test-tmp
14:09:58.849 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
14:09:58.850 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskBD8el9/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
14:09:58.852 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443"
extra_master_flags: "--webserver_require_spnego=false"
extra_master_flags: "--authn_token_validity_seconds=10"
extra_tserver_flags: "--webserver_require_spnego=false"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
14:09:59.189 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:09:59.189 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
14:09:59.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:59.193830 21310 mini_oidc.cc:150] Starting JWKS server
14:09:59.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:59.197268 21310 webserver.cc:480] Webserver started at http://127.0.0.1:34701/ using document root <none> and password file <none>
14:09:59.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:59.197769 21310 mini_oidc.cc:186] Starting OIDC Discovery server
14:09:59.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:59.199539 21310 webserver.cc:480] Webserver started at http://127.0.0.1:35739/ using document root <none> and password file <none>
14:09:59.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
14:09:59.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/krb5kdc/principal' for realm 'KRBTEST.COM',
14:09:59.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
14:09:59.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 30 14:09:59 dist-test-slave-xkqm krb5kdc[21321](info): setting up network...
14:09:59.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
14:09:59.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 30 14:09:59 dist-test-slave-xkqm krb5kdc[21321](info): set up 2 sockets
14:09:59.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 30 14:09:59 dist-test-slave-xkqm krb5kdc[21321](info): commencing operation
14:09:59.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
14:09:59.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:09:59.817727 21303 heartbeater.cc:499] Master 127.20.25.254:39857 was elected leader, sending a full tablet report...
14:10:01.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:01.347810 21310 mini_kdc.cc:121] Time spent starting KDC: real 2.110s user 0.003s sys 0.016s
14:10:01.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
14:10:01.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
14:10:01.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
14:10:01.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
14:10:01.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
14:10:01.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
14:10:01.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
14:10:01.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
14:10:01.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
14:10:01.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
14:10:01.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/krb5kdc/test-user.keytab.
14:10:01.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/krb5kdc/test-user.keytab.
14:10:01.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 30 14:10:01 dist-test-slave-xkqm krb5kdc[21321](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756563001, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:10:01.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
14:10:01.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-08-30T14:10:01Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
14:10:01.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-08-30T14:10:01Z Disabled control of system clock
14:10:01.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.20.207.190@KRBTEST.COM; defaulting to no policy
14:10:01.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:10:01.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.20.207.190@KRBTEST.COM" created.
14:10:01.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:10:01.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.20.207.190 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/krb5kdc/kudu.keytab.
14:10:01.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.20.207.190 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/krb5kdc/kudu.keytab.
14:10:01.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.20.207.190@KRBTEST.COM; defaulting to no policy
14:10:01.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:10:01.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.20.207.190@KRBTEST.COM" created.
14:10:01.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:10:01.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.20.207.190 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/krb5kdc/kudu.keytab.
14:10:01.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.20.207.190 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/krb5kdc/kudu.keytab.
14:10:01.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:01.592418 21310 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskBD8el9/build/tsan/bin/kudu
14:10:01.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskBD8el9/build/tsan/bin/kudu
14:10:01.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-0/wal
14:10:01.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-0/data
14:10:01.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:10:01.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:10:01.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:10:01.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:10:01.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:10:01.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:10:01.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-0/logs
14:10:01.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-0/data/info.pb
14:10:01.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:10:01.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:10:01.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:10:01.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:10:01.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:10:01.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:10:01.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:10:01.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:10:01.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:10:01.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:10:01.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:10:01.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:10:01.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.207.190:38271
14:10:01.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.207.190
14:10:01.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:10:01.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/krb5kdc/kudu.keytab
14:10:01.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.20.207.190
14:10:01.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:10:01.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:10:01.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:10:01.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
14:10:01.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.207.148:36627
14:10:01.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:10:01.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:10:01.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:10:01.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:10:01.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.207.190:38271,127.20.207.189:43023,127.20.207.188:42709
14:10:01.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:10:01.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34701/jwks.json
14:10:01.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
14:10:01.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:10:01.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:01.913626 21339 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:10:01.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:01.914285 21339 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:10:01.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:01.915015 21339 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:10:01.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:01.948773 21339 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
14:10:01.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:01.949048 21339 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:10:01.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:01.949525 21339 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:10:01.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:01.949827 21339 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:10:01.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:01.950119 21339 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:10:01.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:01.950388 21339 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:10:01.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:01.950609 21339 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34701/jwks.json
14:10:01.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:01.988325 21339 master_runner.cc:386] Master server non-default flags:
14:10:01.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:10:01.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.207.148:36627
14:10:01.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:10:01.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:10:01.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-0/data
14:10:01.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-0/wal
14:10:01.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
14:10:01.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:10:01.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.207.190:38271,127.20.207.189:43023,127.20.207.188:42709
14:10:01.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/krb5kdc/kudu.keytab
14:10:01.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.20.207.190
14:10:01.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:10:01.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:10:01.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:10:01.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.207.190:38271
14:10:01.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:10:01.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:10:01.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:10:01.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34701/jwks.json
14:10:01.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:10:01.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:10:01.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:10:01.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:10:01.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:10:01.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-0/data/info.pb
14:10:01.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.207.190
14:10:01.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:10:01.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:10:01.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:10:01.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:10:01.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:10:01.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:10:01.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-0/logs
14:10:01.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:10:01.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:10:01.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:10:01.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:10:01.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:10:01.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 06466b25e294b4d7d455e6c56263318dc34a80ef
14:10:01.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:10:01.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 30 Aug 2025 13:43:24 UTC on 5fd53c4cbb9d
14:10:01.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7817
14:10:01.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:10:01.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:01.989976 21339 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:10:01.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:01.991170 21339 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:10:02.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:02.002041 21345 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:10:03.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:02.002893 21346 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:10:03.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:03.410506 21344 debug-util.cc:398] Leaking SignalData structure 0x7b0800037200 after lost signal to thread 21339
14:10:03.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:03.496991 21344 kernel_stack_watchdog.cc:198] Thread 21339 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 404ms:
14:10:03.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
14:10:03.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
14:10:03.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
14:10:03.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
14:10:03.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:03.498231 21339 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.496s user 0.468s sys 1.009s
14:10:03.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:03.498541 21339 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.496s user 0.468s sys 1.009s
14:10:03.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:03.499763 21348 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:10:03.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:03.501410 21347 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1494 milliseconds
14:10:03.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:03.501473 21339 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:10:03.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:03.502830 21339 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:10:03.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:03.505215 21339 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:10:03.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:03.506565 21339 hybrid_clock.cc:648] HybridClock initialized: now 1756563003506532 us; error 40 us; skew 500 ppm
14:10:03.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:03.507355 21339 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:10:03.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 30 14:10:03 dist-test-slave-xkqm krb5kdc[21321](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756563003, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.207.190@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:10:03.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:03.512921 21339 init.cc:377] Logged in from keytab as kudu/127.20.207.190@KRBTEST.COM (short username kudu)
14:10:03.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:03.517506 21339 webserver.cc:480] Webserver started at http://127.20.207.190:37257/ using document root <none> and password file <none>
14:10:03.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:03.518647 21339 fs_manager.cc:362] Metadata directory not provided
14:10:03.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:03.518920 21339 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:10:03.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:03.519404 21339 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
14:10:03.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:03.524169 21339 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-0/data/instance:
14:10:03.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f34e5ae5629346b380aa3975ffecb1f3"
14:10:03.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-30 14:10:03 on dist-test-slave-xkqm"
14:10:03.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:03.525687 21339 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-0/wal/instance:
14:10:03.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f34e5ae5629346b380aa3975ffecb1f3"
14:10:03.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-30 14:10:03 on dist-test-slave-xkqm"
14:10:03.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:03.533319 21339 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.002s sys 0.006s
14:10:03.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:03.539011 21357 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:10:03.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:03.540143 21339 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.001s sys 0.001s
14:10:03.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:03.540546 21339 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-0/data,/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-0/wal
14:10:03.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f34e5ae5629346b380aa3975ffecb1f3"
14:10:03.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-30 14:10:03 on dist-test-slave-xkqm"
14:10:03.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:03.540849 21339 fs_report.cc:389] FS layout report
14:10:03.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:10:03.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-0/wal
14:10:03.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-0/wal
14:10:03.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-0/data/data
14:10:03.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:10:03.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:10:03.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:10:03.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:10:03.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:10:03.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:10:03.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:10:03.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:10:03.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:10:03.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:10:03.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:10:03.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:03.602279 21339 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:10:03.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:03.607754 21339 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:10:03.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:03.608152 21339 kserver.cc:163] Server-wide thread pool size limit: 3276
14:10:03.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:03.683353 21339 rpc_server.cc:307] RPC server started. Bound to: 127.20.207.190:38271
14:10:03.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:03.683457 21408 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.20.207.190:38271 every 8 connection(s)
14:10:03.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:03.686084 21339 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-0/data/info.pb
14:10:03.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:03.691323 21409 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
14:10:03.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:03.692641 21310 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskBD8el9/build/tsan/bin/kudu as pid 21339
14:10:03.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:03.693013 21310 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-0/wal/instance
14:10:03.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:03.703809 21409 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.207.190" port: 38271 } has no permanent_uuid. Determining permanent_uuid...
14:10:03.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.20.207.189@KRBTEST.COM; defaulting to no policy
14:10:03.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:10:03.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.20.207.189@KRBTEST.COM" created.
14:10:03.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 30 14:10:03 dist-test-slave-xkqm krb5kdc[21321](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756563003, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.207.190@KRBTEST.COM for kudu/127.20.207.190@KRBTEST.COM
14:10:03.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:10:03.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.20.207.189 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/krb5kdc/kudu.keytab.
14:10:03.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.20.207.189 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/krb5kdc/kudu.keytab.
14:10:03.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:03.749702 21409 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.207.189" port: 43023 } has no permanent_uuid. Determining permanent_uuid...
14:10:03.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:03.752525 21361 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.20.207.189:43023: connect: Connection refused (error 111)
14:10:03.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.20.207.189@KRBTEST.COM; defaulting to no policy
14:10:03.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:10:03.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.20.207.189@KRBTEST.COM" created.
14:10:03.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:03.762226 21409 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.20.207.189:43023: Network error: Client connection negotiation failed: client connection to 127.20.207.189:43023: connect: Connection refused (error 111)
14:10:03.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:10:03.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.20.207.189 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/krb5kdc/kudu.keytab.
14:10:03.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.20.207.189 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/krb5kdc/kudu.keytab.
14:10:03.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:03.785173 21310 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskBD8el9/build/tsan/bin/kudu
14:10:03.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskBD8el9/build/tsan/bin/kudu
14:10:03.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-1/wal
14:10:03.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-1/data
14:10:03.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:10:03.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:10:03.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:10:03.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:10:03.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:10:03.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:10:03.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-1/logs
14:10:03.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-1/data/info.pb
14:10:03.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:10:03.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:10:03.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:10:03.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:10:03.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:10:03.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:10:03.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:10:03.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:10:03.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:10:03.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:10:03.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:10:03.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:10:03.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.207.189:43023
14:10:03.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.207.189
14:10:03.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:10:03.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/krb5kdc/kudu.keytab
14:10:03.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.20.207.189
14:10:03.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:10:03.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:10:03.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:10:03.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
14:10:03.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.207.148:36627
14:10:03.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:10:03.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:10:03.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:10:03.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:10:03.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.207.190:38271,127.20.207.189:43023,127.20.207.188:42709
14:10:03.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:10:03.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34701/jwks.json
14:10:03.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
14:10:03.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:10:03.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:03.811761 21409 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.207.189" port: 43023 } attempt: 1
14:10:03.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:03.815996 21409 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.20.207.189:43023: Network error: Client connection negotiation failed: client connection to 127.20.207.189:43023: connect: Connection refused (error 111)
14:10:03.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:03.884456 21409 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.207.189" port: 43023 } attempt: 2
14:10:03.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:03.888470 21409 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.20.207.189:43023: Network error: Client connection negotiation failed: client connection to 127.20.207.189:43023: connect: Connection refused (error 111)
14:10:03.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:03.979874 21409 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.207.189" port: 43023 } attempt: 3
14:10:03.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:03.984021 21409 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.20.207.189:43023: Network error: Client connection negotiation failed: client connection to 127.20.207.189:43023: connect: Connection refused (error 111)
14:10:04.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:04.119194 21419 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:10:04.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:04.119824 21419 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:10:04.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:04.120625 21419 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:10:04.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:04.127467 21409 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.207.189" port: 43023 } attempt: 4
14:10:04.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:04.131786 21409 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.20.207.189:43023: Network error: Client connection negotiation failed: client connection to 127.20.207.189:43023: connect: Connection refused (error 111)
14:10:04.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:04.156536 21419 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
14:10:04.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:04.156778 21419 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:10:04.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:04.157040 21419 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:10:04.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:04.157238 21419 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:10:04.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:04.157439 21419 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:10:04.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:04.157627 21419 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:10:04.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:04.157771 21419 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34701/jwks.json
14:10:04.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:04.197767 21419 master_runner.cc:386] Master server non-default flags:
14:10:04.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:10:04.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.207.148:36627
14:10:04.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:10:04.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:10:04.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-1/data
14:10:04.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-1/wal
14:10:04.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
14:10:04.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:10:04.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.207.190:38271,127.20.207.189:43023,127.20.207.188:42709
14:10:04.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/krb5kdc/kudu.keytab
14:10:04.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.20.207.189
14:10:04.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:10:04.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:10:04.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:10:04.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.207.189:43023
14:10:04.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:10:04.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:10:04.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:10:04.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34701/jwks.json
14:10:04.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:10:04.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:10:04.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:10:04.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:10:04.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:10:04.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-1/data/info.pb
14:10:04.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.207.189
14:10:04.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:10:04.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:10:04.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:10:04.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:10:04.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:10:04.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:10:04.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-1/logs
14:10:04.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:10:04.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:10:04.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:10:04.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:10:04.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:10:04.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 06466b25e294b4d7d455e6c56263318dc34a80ef
14:10:04.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:10:04.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 30 Aug 2025 13:43:24 UTC on 5fd53c4cbb9d
14:10:04.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7817
14:10:04.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:10:04.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:04.199090 21419 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:10:04.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:04.200105 21419 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:10:04.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:04.210889 21427 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:10:04.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:04.431193 21409 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.207.189" port: 43023 } attempt: 5
14:10:04.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:04.439270 21409 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.20.207.189:43023: Network error: Client connection negotiation failed: client connection to 127.20.207.189:43023: connect: Connection refused (error 111)
14:10:04.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:04.986872 21409 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.207.189" port: 43023 } attempt: 6
14:10:04.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:04.998129 21409 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.20.207.189:43023: Network error: Client connection negotiation failed: client connection to 127.20.207.189:43023: connect: Connection refused (error 111)
14:10:05.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:04.212988 21428 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:10:05.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:05.566203 21430 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:10:05.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:05.568670 21429 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1352 milliseconds
14:10:05.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:05.568842 21419 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:10:05.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:05.570382 21419 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:10:05.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:05.573395 21419 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:10:05.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:05.574832 21419 hybrid_clock.cc:648] HybridClock initialized: now 1756563005574788 us; error 49 us; skew 500 ppm
14:10:05.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:05.575675 21419 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:10:05.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 30 14:10:05 dist-test-slave-xkqm krb5kdc[21321](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756563005, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.207.189@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:10:05.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:05.581447 21419 init.cc:377] Logged in from keytab as kudu/127.20.207.189@KRBTEST.COM (short username kudu)
14:10:05.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:05.586761 21419 webserver.cc:480] Webserver started at http://127.20.207.189:43607/ using document root <none> and password file <none>
14:10:05.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:05.587715 21419 fs_manager.cc:362] Metadata directory not provided
14:10:05.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:05.587939 21419 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:10:05.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:05.588445 21419 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
14:10:05.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:05.592836 21419 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-1/data/instance:
14:10:05.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1fd5034219e348e289cf311cecde23f1"
14:10:05.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-30 14:10:05 on dist-test-slave-xkqm"
14:10:05.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:05.593984 21419 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-1/wal/instance:
14:10:05.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1fd5034219e348e289cf311cecde23f1"
14:10:05.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-30 14:10:05 on dist-test-slave-xkqm"
14:10:05.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:05.601274 21419 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.008s sys 0.000s
14:10:05.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:05.606870 21441 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:10:05.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:05.607962 21419 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.002s sys 0.000s
14:10:05.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:05.608309 21419 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-1/data,/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-1/wal
14:10:05.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1fd5034219e348e289cf311cecde23f1"
14:10:05.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-30 14:10:05 on dist-test-slave-xkqm"
14:10:05.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:05.608625 21419 fs_report.cc:389] FS layout report
14:10:05.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:10:05.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-1/wal
14:10:05.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-1/wal
14:10:05.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-1/data/data
14:10:05.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:10:05.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:10:05.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:10:05.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:10:05.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:10:05.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:10:05.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:10:05.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:10:05.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:10:05.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:10:05.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:10:05.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:05.662074 21419 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:10:05.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:05.668210 21419 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:10:05.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:05.668668 21419 kserver.cc:163] Server-wide thread pool size limit: 3276
14:10:05.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:05.748623 21419 rpc_server.cc:307] RPC server started. Bound to: 127.20.207.189:43023
14:10:05.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:05.748674 21492 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.20.207.189:43023 every 8 connection(s)
14:10:05.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:05.751763 21419 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-1/data/info.pb
14:10:05.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:05.758679 21493 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
14:10:05.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:05.760071 21310 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskBD8el9/build/tsan/bin/kudu as pid 21419
14:10:05.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:05.760555 21310 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-1/wal/instance
14:10:05.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:05.773002 21493 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.207.190" port: 38271 } has no permanent_uuid. Determining permanent_uuid...
14:10:05.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.20.207.188@KRBTEST.COM; defaulting to no policy
14:10:05.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:10:05.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.20.207.188@KRBTEST.COM" created.
14:10:05.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 30 14:10:05 dist-test-slave-xkqm krb5kdc[21321](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756563005, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.207.189@KRBTEST.COM for kudu/127.20.207.190@KRBTEST.COM
14:10:05.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:10:05.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.20.207.188 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/krb5kdc/kudu.keytab.
14:10:05.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.20.207.188 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/krb5kdc/kudu.keytab.
14:10:05.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:05.818773 21493 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.207.189" port: 43023 } has no permanent_uuid. Determining permanent_uuid...
14:10:05.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.20.207.188@KRBTEST.COM; defaulting to no policy
14:10:05.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:10:05.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.20.207.188@KRBTEST.COM" created.
14:10:05.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 30 14:10:05 dist-test-slave-xkqm krb5kdc[21321](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756563005, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.207.189@KRBTEST.COM for kudu/127.20.207.189@KRBTEST.COM
14:10:05.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:05.845324 21493 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.207.188" port: 42709 } has no permanent_uuid. Determining permanent_uuid...
14:10:05.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:05.847774 21444 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.20.207.188:42709: connect: Connection refused (error 111)
14:10:05.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:05.852789 21493 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.20.207.188:42709: Network error: Client connection negotiation failed: client connection to 127.20.207.188:42709: connect: Connection refused (error 111)
14:10:05.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:10:05.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.20.207.188 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/krb5kdc/kudu.keytab.
14:10:05.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.20.207.188 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/krb5kdc/kudu.keytab.
14:10:05.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:05.856863 21310 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskBD8el9/build/tsan/bin/kudu
14:10:05.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskBD8el9/build/tsan/bin/kudu
14:10:05.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-2/wal
14:10:05.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-2/data
14:10:05.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:10:05.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:10:05.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:10:05.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:10:05.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:10:05.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:10:05.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-2/logs
14:10:05.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-2/data/info.pb
14:10:05.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:10:05.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:10:05.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:10:05.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:10:05.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:10:05.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:10:05.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:10:05.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:10:05.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:10:05.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:10:05.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:10:05.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:10:05.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.207.188:42709
14:10:05.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.207.188
14:10:05.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:10:05.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/krb5kdc/kudu.keytab
14:10:05.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.20.207.188
14:10:05.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:10:05.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:10:05.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:10:05.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
14:10:05.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.207.148:36627
14:10:05.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:10:05.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:10:05.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:10:05.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:10:05.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.207.190:38271,127.20.207.189:43023,127.20.207.188:42709
14:10:05.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:10:05.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34701/jwks.json
14:10:05.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
14:10:05.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:10:05.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:05.902348 21493 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.207.188" port: 42709 } attempt: 1
14:10:05.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:05.906221 21493 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.20.207.188:42709: Network error: Client connection negotiation failed: client connection to 127.20.207.188:42709: connect: Connection refused (error 111)
14:10:05.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:05.974685 21493 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.207.188" port: 42709 } attempt: 2
14:10:05.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:05.978549 21493 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.20.207.188:42709: Network error: Client connection negotiation failed: client connection to 127.20.207.188:42709: connect: Connection refused (error 111)
14:10:06.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:06.058744 21409 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.207.189" port: 43023 } attempt: 7
14:10:06.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 30 14:10:06 dist-test-slave-xkqm krb5kdc[21321](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756563003, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.207.190@KRBTEST.COM for kudu/127.20.207.189@KRBTEST.COM
14:10:06.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:06.070013 21493 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.207.188" port: 42709 } attempt: 3
14:10:06.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:06.074844 21493 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.20.207.188:42709: Network error: Client connection negotiation failed: client connection to 127.20.207.188:42709: connect: Connection refused (error 111)
14:10:06.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:06.082136 21409 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.207.188" port: 42709 } has no permanent_uuid. Determining permanent_uuid...
14:10:06.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:06.087579 21409 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.20.207.188:42709: Network error: Client connection negotiation failed: client connection to 127.20.207.188:42709: connect: Connection refused (error 111)
14:10:06.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:06.145927 21409 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.207.188" port: 42709 } attempt: 1
14:10:06.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:06.149772 21409 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.20.207.188:42709: Network error: Client connection negotiation failed: client connection to 127.20.207.188:42709: connect: Connection refused (error 111)
14:10:06.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:06.173389 21503 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:10:06.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:06.173959 21503 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:10:06.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:06.174621 21503 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:10:06.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:06.206279 21503 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
14:10:06.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:06.206539 21503 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:10:06.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:06.206804 21503 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:10:06.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:06.206987 21503 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:10:06.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:06.207181 21503 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:10:06.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:06.207377 21503 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:10:06.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:06.207532 21503 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34701/jwks.json
14:10:06.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:06.218258 21493 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.207.188" port: 42709 } attempt: 4
14:10:06.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:06.222694 21493 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.20.207.188:42709: Network error: Client connection negotiation failed: client connection to 127.20.207.188:42709: connect: Connection refused (error 111)
14:10:06.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:06.231279 21409 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.207.188" port: 42709 } attempt: 2
14:10:06.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:06.235770 21409 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.20.207.188:42709: Network error: Client connection negotiation failed: client connection to 127.20.207.188:42709: connect: Connection refused (error 111)
14:10:06.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:06.249114 21503 master_runner.cc:386] Master server non-default flags:
14:10:06.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:10:06.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.207.148:36627
14:10:06.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:10:06.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:10:06.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-2/data
14:10:06.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-2/wal
14:10:06.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
14:10:06.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:10:06.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.207.190:38271,127.20.207.189:43023,127.20.207.188:42709
14:10:06.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/krb5kdc/kudu.keytab
14:10:06.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.20.207.188
14:10:06.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:10:06.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:10:06.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:10:06.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.207.188:42709
14:10:06.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:10:06.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:10:06.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:10:06.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34701/jwks.json
14:10:06.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:10:06.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:10:06.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:10:06.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:10:06.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:10:06.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-2/data/info.pb
14:10:06.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.207.188
14:10:06.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:10:06.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:10:06.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:10:06.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:10:06.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:10:06.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:10:06.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-2/logs
14:10:06.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:10:06.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:10:06.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:10:06.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:10:06.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:10:06.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 06466b25e294b4d7d455e6c56263318dc34a80ef
14:10:06.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:10:06.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 30 Aug 2025 13:43:24 UTC on 5fd53c4cbb9d
14:10:06.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7817
14:10:06.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:10:06.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:06.250488 21503 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:10:06.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:06.251518 21503 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:10:06.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:06.263571 21511 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:10:06.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:06.321270 21409 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.207.188" port: 42709 } attempt: 3
14:10:06.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:06.329520 21409 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.20.207.188:42709: Network error: Client connection negotiation failed: client connection to 127.20.207.188:42709: connect: Connection refused (error 111)
14:10:06.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:06.470117 21409 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.207.188" port: 42709 } attempt: 4
14:10:06.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:06.480469 21409 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.20.207.188:42709: Network error: Client connection negotiation failed: client connection to 127.20.207.188:42709: connect: Connection refused (error 111)
14:10:06.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:06.522117 21493 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.207.188" port: 42709 } attempt: 5
14:10:06.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:06.529536 21493 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.20.207.188:42709: Network error: Client connection negotiation failed: client connection to 127.20.207.188:42709: connect: Connection refused (error 111)
14:10:06.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:06.764118 21409 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.207.188" port: 42709 } attempt: 5
14:10:06.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:06.772454 21409 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.20.207.188:42709: Network error: Client connection negotiation failed: client connection to 127.20.207.188:42709: connect: Connection refused (error 111)
14:10:07.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:07.077108 21493 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.207.188" port: 42709 } attempt: 6
14:10:07.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:07.112670 21493 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.20.207.188:42709: Network error: Client connection negotiation failed: client connection to 127.20.207.188:42709: connect: Connection refused (error 111)
14:10:07.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:07.324913 21409 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.207.188" port: 42709 } attempt: 6
14:10:07.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:07.356614 21409 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.20.207.188:42709: Network error: Client connection negotiation failed: client connection to 127.20.207.188:42709: connect: Connection refused (error 111)
14:10:07.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:07.667960 21510 debug-util.cc:398] Leaking SignalData structure 0x7b0800037200 after lost signal to thread 21503
14:10:07.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:07.933548 21510 kernel_stack_watchdog.cc:198] Thread 21503 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 401ms:
14:10:07.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
14:10:07.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
14:10:07.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
14:10:07.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
14:10:07.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:06.264132 21512 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:10:07.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:07.934623 21503 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.672s user 0.590s sys 1.052s
14:10:07.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:07.935087 21503 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.672s user 0.590s sys 1.052s
14:10:07.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:07.935891 21514 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:10:07.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:07.938542 21513 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1670 milliseconds
14:10:07.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:07.938614 21503 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:10:07.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:07.940003 21503 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:10:07.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:07.942503 21503 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:10:07.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:07.944043 21503 hybrid_clock.cc:648] HybridClock initialized: now 1756563007943993 us; error 56 us; skew 500 ppm
14:10:07.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:07.945026 21503 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:10:07.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 30 14:10:07 dist-test-slave-xkqm krb5kdc[21321](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756563007, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.207.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:10:07.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:07.951139 21503 init.cc:377] Logged in from keytab as kudu/127.20.207.188@KRBTEST.COM (short username kudu)
14:10:07.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:07.957108 21503 webserver.cc:480] Webserver started at http://127.20.207.188:38937/ using document root <none> and password file <none>
14:10:07.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:07.958073 21503 fs_manager.cc:362] Metadata directory not provided
14:10:07.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:07.958308 21503 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:10:07.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:07.958803 21503 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
14:10:07.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:07.963243 21503 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-2/data/instance:
14:10:07.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9c6aef56265e4e5ca5ee68e82eba93c9"
14:10:07.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-30 14:10:07 on dist-test-slave-xkqm"
14:10:07.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:07.964705 21503 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-2/wal/instance:
14:10:07.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9c6aef56265e4e5ca5ee68e82eba93c9"
14:10:07.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-30 14:10:07 on dist-test-slave-xkqm"
14:10:07.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:07.972324 21503 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.007s sys 0.000s
14:10:07.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:07.978020 21530 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:10:07.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:07.979039 21503 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
14:10:07.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:07.979401 21503 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-2/data,/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-2/wal
14:10:07.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9c6aef56265e4e5ca5ee68e82eba93c9"
14:10:07.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-30 14:10:07 on dist-test-slave-xkqm"
14:10:07.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:07.979722 21503 fs_report.cc:389] FS layout report
14:10:07.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:10:07.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-2/wal
14:10:07.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-2/wal
14:10:07.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-2/data/data
14:10:07.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:10:07.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:10:07.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:10:07.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:10:07.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:10:07.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:10:07.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:10:07.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:10:07.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:10:07.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:10:07.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:10:08.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:08.032112 21503 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:10:08.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:08.038513 21503 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:10:08.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:08.039052 21503 kserver.cc:163] Server-wide thread pool size limit: 3276
14:10:08.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:08.112844 21503 rpc_server.cc:307] RPC server started. Bound to: 127.20.207.188:42709
14:10:08.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:08.112911 21581 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.20.207.188:42709 every 8 connection(s)
14:10:08.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:08.115608 21503 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-2/data/info.pb
14:10:08.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:08.118853 21310 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskBD8el9/build/tsan/bin/kudu as pid 21503
14:10:08.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:08.119333 21310 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-2/wal/instance
14:10:08.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:08.121740 21582 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
14:10:08.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:08.146251 21582 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.207.190" port: 38271 } has no permanent_uuid. Determining permanent_uuid...
14:10:08.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 30 14:10:08 dist-test-slave-xkqm krb5kdc[21321](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756563001, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.20.207.188@KRBTEST.COM
14:10:08.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 30 14:10:08 dist-test-slave-xkqm krb5kdc[21321](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756563007, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.207.188@KRBTEST.COM for kudu/127.20.207.190@KRBTEST.COM
14:10:08.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:08.173219 21493 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.207.188" port: 42709 } attempt: 7
14:10:08.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:08.179028 21582 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.207.189" port: 43023 } has no permanent_uuid. Determining permanent_uuid...
14:10:08.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 30 14:10:08 dist-test-slave-xkqm krb5kdc[21321](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756563001, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.20.207.189@KRBTEST.COM
14:10:08.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 30 14:10:08 dist-test-slave-xkqm krb5kdc[21321](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756563005, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.207.189@KRBTEST.COM for kudu/127.20.207.188@KRBTEST.COM
14:10:08.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 30 14:10:08 dist-test-slave-xkqm krb5kdc[21321](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756563007, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.207.188@KRBTEST.COM for kudu/127.20.207.189@KRBTEST.COM
14:10:08.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 30 14:10:08 dist-test-slave-xkqm krb5kdc[21321](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756563001, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.20.207.190@KRBTEST.COM
14:10:08.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:08.213238 21493 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1fd5034219e348e289cf311cecde23f1: Bootstrap starting.
14:10:08.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:08.213565 21582 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.207.188" port: 42709 } has no permanent_uuid. Determining permanent_uuid...
14:10:08.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:08.220713 21493 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 1fd5034219e348e289cf311cecde23f1: Neither blocks nor log segments found. Creating new log.
14:10:08.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:08.223623 21493 log.cc:826] T 00000000000000000000000000000000 P 1fd5034219e348e289cf311cecde23f1: Log is configured to *not* fsync() on all Append() calls
14:10:08.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:08.230549 21310 external_mini_cluster.cc:949] 0 TS(s) registered with all masters
14:10:08.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 30 14:10:08 dist-test-slave-xkqm krb5kdc[21321](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756563007, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.207.188@KRBTEST.COM for kudu/127.20.207.188@KRBTEST.COM
14:10:08.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:08.235224 21493 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1fd5034219e348e289cf311cecde23f1: No bootstrap required, opened a new log
14:10:08.241 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:10:08.242 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
14:10:08.243 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
14:10:08.243 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
14:10:08.245 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.20.207.190"
port: 38271
}
bound_http_address {
host: "127.20.207.190"
port: 37257
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.20.207.189"
port: 43023
}
bound_http_address {
host: "127.20.207.189"
port: 43607
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.20.207.188"
port: 42709
}
bound_http_address {
host: "127.20.207.188"
port: 38937
}
}
}
14:10:08.245 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
14:10:08.248 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:10:08.248 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
14:10:08.248 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
14:10:08.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:08.250476 21582 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9c6aef56265e4e5ca5ee68e82eba93c9: Bootstrap starting.
14:10:08.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:08.256192 21582 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 9c6aef56265e4e5ca5ee68e82eba93c9: Neither blocks nor log segments found. Creating new log.
14:10:08.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:08.258683 21582 log.cc:826] T 00000000000000000000000000000000 P 9c6aef56265e4e5ca5ee68e82eba93c9: Log is configured to *not* fsync() on all Append() calls
14:10:08.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:08.260834 21493 raft_consensus.cc:357] T 00000000000000000000000000000000 P 1fd5034219e348e289cf311cecde23f1 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f34e5ae5629346b380aa3975ffecb1f3" member_type: VOTER last_known_addr { host: "127.20.207.190" port: 38271 } } peers { permanent_uuid: "1fd5034219e348e289cf311cecde23f1" member_type: VOTER last_known_addr { host: "127.20.207.189" port: 43023 } } peers { permanent_uuid: "9c6aef56265e4e5ca5ee68e82eba93c9" member_type: VOTER last_known_addr { host: "127.20.207.188" port: 42709 } }
14:10:08.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:08.261754 21493 raft_consensus.cc:383] T 00000000000000000000000000000000 P 1fd5034219e348e289cf311cecde23f1 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:10:08.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:08.262076 21493 raft_consensus.cc:738] T 00000000000000000000000000000000 P 1fd5034219e348e289cf311cecde23f1 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1fd5034219e348e289cf311cecde23f1, State: Initialized, Role: FOLLOWER
14:10:08.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:08.262976 21493 consensus_queue.cc:260] T 00000000000000000000000000000000 P 1fd5034219e348e289cf311cecde23f1 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f34e5ae5629346b380aa3975ffecb1f3" member_type: VOTER last_known_addr { host: "127.20.207.190" port: 38271 } } peers { permanent_uuid: "1fd5034219e348e289cf311cecde23f1" member_type: VOTER last_known_addr { host: "127.20.207.189" port: 43023 } } peers { permanent_uuid: "9c6aef56265e4e5ca5ee68e82eba93c9" member_type: VOTER last_known_addr { host: "127.20.207.188" port: 42709 } }
14:10:08.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:08.265022 21582 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9c6aef56265e4e5ca5ee68e82eba93c9: No bootstrap required, opened a new log
14:10:08.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:08.266294 21595 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1fd5034219e348e289cf311cecde23f1 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f34e5ae5629346b380aa3975ffecb1f3" member_type: VOTER last_known_addr { host: "127.20.207.190" port: 38271 } } peers { permanent_uuid: "1fd5034219e348e289cf311cecde23f1" member_type: VOTER last_known_addr { host: "127.20.207.189" port: 43023 } } peers { permanent_uuid: "9c6aef56265e4e5ca5ee68e82eba93c9" member_type: VOTER last_known_addr { host: "127.20.207.188" port: 42709 } } }
14:10:08.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:08.267256 21595 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1fd5034219e348e289cf311cecde23f1 [sys.catalog]: This master's current role is: FOLLOWER
14:10:08.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:08.268203 21493 sys_catalog.cc:564] T 00000000000000000000000000000000 P 1fd5034219e348e289cf311cecde23f1 [sys.catalog]: configured and running, proceeding with master startup.
14:10:08.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 30 14:10:08 dist-test-slave-xkqm krb5kdc[21321](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756563008, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:10:08.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
14:10:08.291 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:10:08.293 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/krb5kdc/krb5cc
14:10:08.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:08.293051 21582 raft_consensus.cc:357] T 00000000000000000000000000000000 P 9c6aef56265e4e5ca5ee68e82eba93c9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f34e5ae5629346b380aa3975ffecb1f3" member_type: VOTER last_known_addr { host: "127.20.207.190" port: 38271 } } peers { permanent_uuid: "1fd5034219e348e289cf311cecde23f1" member_type: VOTER last_known_addr { host: "127.20.207.189" port: 43023 } } peers { permanent_uuid: "9c6aef56265e4e5ca5ee68e82eba93c9" member_type: VOTER last_known_addr { host: "127.20.207.188" port: 42709 } }
14:10:08.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:08.294085 21582 raft_consensus.cc:383] T 00000000000000000000000000000000 P 9c6aef56265e4e5ca5ee68e82eba93c9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:10:08.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:08.294399 21582 raft_consensus.cc:738] T 00000000000000000000000000000000 P 9c6aef56265e4e5ca5ee68e82eba93c9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9c6aef56265e4e5ca5ee68e82eba93c9, State: Initialized, Role: FOLLOWER
14:10:08.294 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
14:10:08.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:08.295297 21582 consensus_queue.cc:260] T 00000000000000000000000000000000 P 9c6aef56265e4e5ca5ee68e82eba93c9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f34e5ae5629346b380aa3975ffecb1f3" member_type: VOTER last_known_addr { host: "127.20.207.190" port: 38271 } } peers { permanent_uuid: "1fd5034219e348e289cf311cecde23f1" member_type: VOTER last_known_addr { host: "127.20.207.189" port: 43023 } } peers { permanent_uuid: "9c6aef56265e4e5ca5ee68e82eba93c9" member_type: VOTER last_known_addr { host: "127.20.207.188" port: 42709 } }
14:10:08.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:08.300480 21605 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9c6aef56265e4e5ca5ee68e82eba93c9 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f34e5ae5629346b380aa3975ffecb1f3" member_type: VOTER last_known_addr { host: "127.20.207.190" port: 38271 } } peers { permanent_uuid: "1fd5034219e348e289cf311cecde23f1" member_type: VOTER last_known_addr { host: "127.20.207.189" port: 43023 } } peers { permanent_uuid: "9c6aef56265e4e5ca5ee68e82eba93c9" member_type: VOTER last_known_addr { host: "127.20.207.188" port: 42709 } } }
14:10:08.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:08.301599 21605 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9c6aef56265e4e5ca5ee68e82eba93c9 [sys.catalog]: This master's current role is: FOLLOWER
14:10:08.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:08.303012 21582 sys_catalog.cc:564] T 00000000000000000000000000000000 P 9c6aef56265e4e5ca5ee68e82eba93c9 [sys.catalog]: configured and running, proceeding with master startup.
14:10:08.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:08.313467 21608 catalog_manager.cc:1560] T 00000000000000000000000000000000 P 1fd5034219e348e289cf311cecde23f1: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:10:08.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:08.314080 21608 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:10:08.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:08.309665 21419 master_runner.cc:186] Error getting master registration for 127.20.207.190:38271: OK, instance_id { permanent_uuid: "f34e5ae5629346b380aa3975ffecb1f3" instance_seqno: 1756563003663125 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
14:10:08.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:08.341115 21419 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:10:08.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:08.347215 21620 catalog_manager.cc:1560] T 00000000000000000000000000000000 P 9c6aef56265e4e5ca5ee68e82eba93c9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:10:08.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:08.347625 21620 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:10:08.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:08.335109 21503 master_runner.cc:186] Error getting master registration for 127.20.207.190:38271: OK, instance_id { permanent_uuid: "f34e5ae5629346b380aa3975ffecb1f3" instance_seqno: 1756563003663125 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
14:10:08.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:08.359001 21503 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:10:08.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:08.365558 21595 raft_consensus.cc:491] T 00000000000000000000000000000000 P 1fd5034219e348e289cf311cecde23f1 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:10:08.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:08.366153 21595 raft_consensus.cc:513] T 00000000000000000000000000000000 P 1fd5034219e348e289cf311cecde23f1 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f34e5ae5629346b380aa3975ffecb1f3" member_type: VOTER last_known_addr { host: "127.20.207.190" port: 38271 } } peers { permanent_uuid: "1fd5034219e348e289cf311cecde23f1" member_type: VOTER last_known_addr { host: "127.20.207.189" port: 43023 } } peers { permanent_uuid: "9c6aef56265e4e5ca5ee68e82eba93c9" member_type: VOTER last_known_addr { host: "127.20.207.188" port: 42709 } }
14:10:08.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:08.368870 21595 leader_election.cc:290] T 00000000000000000000000000000000 P 1fd5034219e348e289cf311cecde23f1 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers f34e5ae5629346b380aa3975ffecb1f3 (127.20.207.190:38271), 9c6aef56265e4e5ca5ee68e82eba93c9 (127.20.207.188:42709)
14:10:08.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:08.370685 21557 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "1fd5034219e348e289cf311cecde23f1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9c6aef56265e4e5ca5ee68e82eba93c9" is_pre_election: true
14:10:08.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:08.371747 21557 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 9c6aef56265e4e5ca5ee68e82eba93c9 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 1fd5034219e348e289cf311cecde23f1 in term 0.
14:10:08.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:08.373023 21444 leader_election.cc:304] T 00000000000000000000000000000000 P 1fd5034219e348e289cf311cecde23f1 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1fd5034219e348e289cf311cecde23f1, 9c6aef56265e4e5ca5ee68e82eba93c9; no voters:
14:10:08.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:08.373986 21595 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 1fd5034219e348e289cf311cecde23f1 [term 0 FOLLOWER]: Leader pre-election won for term 1
14:10:08.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:08.374346 21595 raft_consensus.cc:491] T 00000000000000000000000000000000 P 1fd5034219e348e289cf311cecde23f1 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
14:10:08.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:08.374684 21595 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 1fd5034219e348e289cf311cecde23f1 [term 0 FOLLOWER]: Advancing to term 1
14:10:08.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:08.382767 21595 raft_consensus.cc:513] T 00000000000000000000000000000000 P 1fd5034219e348e289cf311cecde23f1 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f34e5ae5629346b380aa3975ffecb1f3" member_type: VOTER last_known_addr { host: "127.20.207.190" port: 38271 } } peers { permanent_uuid: "1fd5034219e348e289cf311cecde23f1" member_type: VOTER last_known_addr { host: "127.20.207.189" port: 43023 } } peers { permanent_uuid: "9c6aef56265e4e5ca5ee68e82eba93c9" member_type: VOTER last_known_addr { host: "127.20.207.188" port: 42709 } }
14:10:08.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:08.385427 21595 leader_election.cc:290] T 00000000000000000000000000000000 P 1fd5034219e348e289cf311cecde23f1 [CANDIDATE]: Term 1 election: Requested vote from peers f34e5ae5629346b380aa3975ffecb1f3 (127.20.207.190:38271), 9c6aef56265e4e5ca5ee68e82eba93c9 (127.20.207.188:42709)
14:10:08.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:08.386946 21557 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "1fd5034219e348e289cf311cecde23f1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9c6aef56265e4e5ca5ee68e82eba93c9"
14:10:08.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:08.387530 21557 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 9c6aef56265e4e5ca5ee68e82eba93c9 [term 0 FOLLOWER]: Advancing to term 1
14:10:08.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:08.390128 21409 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.207.188" port: 42709 } attempt: 7
14:10:08.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:08.395464 21557 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 9c6aef56265e4e5ca5ee68e82eba93c9 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 1fd5034219e348e289cf311cecde23f1 in term 1.
14:10:08.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:08.396629 21444 leader_election.cc:304] T 00000000000000000000000000000000 P 1fd5034219e348e289cf311cecde23f1 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1fd5034219e348e289cf311cecde23f1, 9c6aef56265e4e5ca5ee68e82eba93c9; no voters:
14:10:08.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:08.397648 21595 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 1fd5034219e348e289cf311cecde23f1 [term 1 FOLLOWER]: Leader election won for term 1
14:10:08.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:08.405833 21595 raft_consensus.cc:695] T 00000000000000000000000000000000 P 1fd5034219e348e289cf311cecde23f1 [term 1 LEADER]: Becoming Leader. State: Replica: 1fd5034219e348e289cf311cecde23f1, State: Running, Role: LEADER
14:10:08.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:08.407089 21595 consensus_queue.cc:237] T 00000000000000000000000000000000 P 1fd5034219e348e289cf311cecde23f1 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f34e5ae5629346b380aa3975ffecb1f3" member_type: VOTER last_known_addr { host: "127.20.207.190" port: 38271 } } peers { permanent_uuid: "1fd5034219e348e289cf311cecde23f1" member_type: VOTER last_known_addr { host: "127.20.207.189" port: 43023 } } peers { permanent_uuid: "9c6aef56265e4e5ca5ee68e82eba93c9" member_type: VOTER last_known_addr { host: "127.20.207.188" port: 42709 } }
14:10:08.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:08.421522 21623 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1fd5034219e348e289cf311cecde23f1 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 1fd5034219e348e289cf311cecde23f1. Latest consensus state: current_term: 1 leader_uuid: "1fd5034219e348e289cf311cecde23f1" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f34e5ae5629346b380aa3975ffecb1f3" member_type: VOTER last_known_addr { host: "127.20.207.190" port: 38271 } } peers { permanent_uuid: "1fd5034219e348e289cf311cecde23f1" member_type: VOTER last_known_addr { host: "127.20.207.189" port: 43023 } } peers { permanent_uuid: "9c6aef56265e4e5ca5ee68e82eba93c9" member_type: VOTER last_known_addr { host: "127.20.207.188" port: 42709 } } }
14:10:08.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:08.422412 21623 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1fd5034219e348e289cf311cecde23f1 [sys.catalog]: This master's current role is: LEADER
14:10:08.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:08.427040 21447 tablet.cc:2378] T 00000000000000000000000000000000 P 1fd5034219e348e289cf311cecde23f1: Can't schedule compaction. Clean time has not been advanced past its initial value.
14:10:08.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 30 14:10:08 dist-test-slave-xkqm krb5kdc[21321](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756563003, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.207.190@KRBTEST.COM for kudu/127.20.207.188@KRBTEST.COM
14:10:08.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:08.443406 21625 catalog_manager.cc:1477] Loading table and tablet metadata into memory...
14:10:08.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:08.452133 21625 catalog_manager.cc:1486] Initializing Kudu cluster ID...
14:10:08.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:08.459396 21409 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f34e5ae5629346b380aa3975ffecb1f3: Bootstrap starting.
14:10:08.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:08.475885 21409 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P f34e5ae5629346b380aa3975ffecb1f3: Neither blocks nor log segments found. Creating new log.
14:10:08.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:08.484035 21557 raft_consensus.cc:1273] T 00000000000000000000000000000000 P 9c6aef56265e4e5ca5ee68e82eba93c9 [term 1 FOLLOWER]: Refusing update from remote peer 1fd5034219e348e289cf311cecde23f1: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
14:10:08.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:08.484304 21409 log.cc:826] T 00000000000000000000000000000000 P f34e5ae5629346b380aa3975ffecb1f3: Log is configured to *not* fsync() on all Append() calls
14:10:08.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:08.486649 21595 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 1fd5034219e348e289cf311cecde23f1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9c6aef56265e4e5ca5ee68e82eba93c9" member_type: VOTER last_known_addr { host: "127.20.207.188" port: 42709 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
14:10:08.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:08.508524 21409 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f34e5ae5629346b380aa3975ffecb1f3: No bootstrap required, opened a new log
14:10:08.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:08.544618 21536 tablet.cc:2378] T 00000000000000000000000000000000 P 9c6aef56265e4e5ca5ee68e82eba93c9: Can't schedule compaction. Clean time has not been advanced past its initial value.
14:10:08.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:08.557333 21605 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9c6aef56265e4e5ca5ee68e82eba93c9 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 1fd5034219e348e289cf311cecde23f1. Latest consensus state: current_term: 1 leader_uuid: "1fd5034219e348e289cf311cecde23f1" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f34e5ae5629346b380aa3975ffecb1f3" member_type: VOTER last_known_addr { host: "127.20.207.190" port: 38271 } } peers { permanent_uuid: "1fd5034219e348e289cf311cecde23f1" member_type: VOTER last_known_addr { host: "127.20.207.189" port: 43023 } } peers { permanent_uuid: "9c6aef56265e4e5ca5ee68e82eba93c9" member_type: VOTER last_known_addr { host: "127.20.207.188" port: 42709 } } }
14:10:08.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:08.560039 21605 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9c6aef56265e4e5ca5ee68e82eba93c9 [sys.catalog]: This master's current role is: FOLLOWER
14:10:08.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:08.568341 21409 raft_consensus.cc:357] T 00000000000000000000000000000000 P f34e5ae5629346b380aa3975ffecb1f3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f34e5ae5629346b380aa3975ffecb1f3" member_type: VOTER last_known_addr { host: "127.20.207.190" port: 38271 } } peers { permanent_uuid: "1fd5034219e348e289cf311cecde23f1" member_type: VOTER last_known_addr { host: "127.20.207.189" port: 43023 } } peers { permanent_uuid: "9c6aef56265e4e5ca5ee68e82eba93c9" member_type: VOTER last_known_addr { host: "127.20.207.188" port: 42709 } }
14:10:08.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:08.569267 21409 raft_consensus.cc:383] T 00000000000000000000000000000000 P f34e5ae5629346b380aa3975ffecb1f3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:10:08.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:08.569573 21409 raft_consensus.cc:738] T 00000000000000000000000000000000 P f34e5ae5629346b380aa3975ffecb1f3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f34e5ae5629346b380aa3975ffecb1f3, State: Initialized, Role: FOLLOWER
14:10:08.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:08.570839 21409 consensus_queue.cc:260] T 00000000000000000000000000000000 P f34e5ae5629346b380aa3975ffecb1f3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f34e5ae5629346b380aa3975ffecb1f3" member_type: VOTER last_known_addr { host: "127.20.207.190" port: 38271 } } peers { permanent_uuid: "1fd5034219e348e289cf311cecde23f1" member_type: VOTER last_known_addr { host: "127.20.207.189" port: 43023 } } peers { permanent_uuid: "9c6aef56265e4e5ca5ee68e82eba93c9" member_type: VOTER last_known_addr { host: "127.20.207.188" port: 42709 } }
14:10:08.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:08.584163 21605 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9c6aef56265e4e5ca5ee68e82eba93c9 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "1fd5034219e348e289cf311cecde23f1" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f34e5ae5629346b380aa3975ffecb1f3" member_type: VOTER last_known_addr { host: "127.20.207.190" port: 38271 } } peers { permanent_uuid: "1fd5034219e348e289cf311cecde23f1" member_type: VOTER last_known_addr { host: "127.20.207.189" port: 43023 } } peers { permanent_uuid: "9c6aef56265e4e5ca5ee68e82eba93c9" member_type: VOTER last_known_addr { host: "127.20.207.188" port: 42709 } } }
14:10:08.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:08.587952 21409 sys_catalog.cc:564] T 00000000000000000000000000000000 P f34e5ae5629346b380aa3975ffecb1f3 [sys.catalog]: configured and running, proceeding with master startup.
14:10:08.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:08.588294 21384 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "1fd5034219e348e289cf311cecde23f1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f34e5ae5629346b380aa3975ffecb1f3" is_pre_election: true
14:10:08.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:08.589172 21384 raft_consensus.cc:2466] T 00000000000000000000000000000000 P f34e5ae5629346b380aa3975ffecb1f3 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 1fd5034219e348e289cf311cecde23f1 in term 0.
14:10:08.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:08.590617 21383 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "1fd5034219e348e289cf311cecde23f1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f34e5ae5629346b380aa3975ffecb1f3"
14:10:08.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:08.579792 21630 sys_catalog.cc:455] T 00000000000000000000000000000000 P f34e5ae5629346b380aa3975ffecb1f3 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f34e5ae5629346b380aa3975ffecb1f3" member_type: VOTER last_known_addr { host: "127.20.207.190" port: 38271 } } peers { permanent_uuid: "1fd5034219e348e289cf311cecde23f1" member_type: VOTER last_known_addr { host: "127.20.207.189" port: 43023 } } peers { permanent_uuid: "9c6aef56265e4e5ca5ee68e82eba93c9" member_type: VOTER last_known_addr { host: "127.20.207.188" port: 42709 } } }
14:10:08.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:08.591239 21383 raft_consensus.cc:3058] T 00000000000000000000000000000000 P f34e5ae5629346b380aa3975ffecb1f3 [term 0 FOLLOWER]: Advancing to term 1
14:10:08.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:08.591428 21630 sys_catalog.cc:458] T 00000000000000000000000000000000 P f34e5ae5629346b380aa3975ffecb1f3 [sys.catalog]: This master's current role is: FOLLOWER
14:10:08.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:08.592200 21625 catalog_manager.cc:1349] Generated new cluster ID: d49b6b05d5d64b61a603c9b5cbaf4320
14:10:08.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:08.592507 21625 catalog_manager.cc:1497] Initializing Kudu internal certificate authority...
14:10:08.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:08.598163 21605 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9c6aef56265e4e5ca5ee68e82eba93c9 [sys.catalog]: This master's current role is: FOLLOWER
14:10:08.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:08.605373 21595 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1fd5034219e348e289cf311cecde23f1 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "1fd5034219e348e289cf311cecde23f1" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f34e5ae5629346b380aa3975ffecb1f3" member_type: VOTER last_known_addr { host: "127.20.207.190" port: 38271 } } peers { permanent_uuid: "1fd5034219e348e289cf311cecde23f1" member_type: VOTER last_known_addr { host: "127.20.207.189" port: 43023 } } peers { permanent_uuid: "9c6aef56265e4e5ca5ee68e82eba93c9" member_type: VOTER last_known_addr { host: "127.20.207.188" port: 42709 } } }
14:10:08.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:08.606129 21595 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1fd5034219e348e289cf311cecde23f1 [sys.catalog]: This master's current role is: LEADER
14:10:08.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:08.606382 21383 raft_consensus.cc:2466] T 00000000000000000000000000000000 P f34e5ae5629346b380aa3975ffecb1f3 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 1fd5034219e348e289cf311cecde23f1 in term 1.
14:10:08.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:08.608654 21382 raft_consensus.cc:1273] T 00000000000000000000000000000000 P f34e5ae5629346b380aa3975ffecb1f3 [term 1 FOLLOWER]: Refusing update from remote peer 1fd5034219e348e289cf311cecde23f1: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
14:10:08.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:08.617852 21623 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 1fd5034219e348e289cf311cecde23f1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f34e5ae5629346b380aa3975ffecb1f3" member_type: VOTER last_known_addr { host: "127.20.207.190" port: 38271 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
14:10:08.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:08.619436 21363 tablet.cc:2378] T 00000000000000000000000000000000 P f34e5ae5629346b380aa3975ffecb1f3: Can't schedule compaction. Clean time has not been advanced past its initial value.
14:10:08.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:08.658484 21625 catalog_manager.cc:1372] Generated new certificate authority record
14:10:08.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:08.661057 21625 catalog_manager.cc:1506] Loading token signing keys...
14:10:08.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:08.681740 21595 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1fd5034219e348e289cf311cecde23f1 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "1fd5034219e348e289cf311cecde23f1" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f34e5ae5629346b380aa3975ffecb1f3" member_type: VOTER last_known_addr { host: "127.20.207.190" port: 38271 } } peers { permanent_uuid: "1fd5034219e348e289cf311cecde23f1" member_type: VOTER last_known_addr { host: "127.20.207.189" port: 43023 } } peers { permanent_uuid: "9c6aef56265e4e5ca5ee68e82eba93c9" member_type: VOTER last_known_addr { host: "127.20.207.188" port: 42709 } } }
14:10:08.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:08.681962 21630 sys_catalog.cc:455] T 00000000000000000000000000000000 P f34e5ae5629346b380aa3975ffecb1f3 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 1fd5034219e348e289cf311cecde23f1. Latest consensus state: current_term: 1 leader_uuid: "1fd5034219e348e289cf311cecde23f1" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f34e5ae5629346b380aa3975ffecb1f3" member_type: VOTER last_known_addr { host: "127.20.207.190" port: 38271 } } peers { permanent_uuid: "1fd5034219e348e289cf311cecde23f1" member_type: VOTER last_known_addr { host: "127.20.207.189" port: 43023 } } peers { permanent_uuid: "9c6aef56265e4e5ca5ee68e82eba93c9" member_type: VOTER last_known_addr { host: "127.20.207.188" port: 42709 } } }
14:10:08.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:08.682515 21595 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1fd5034219e348e289cf311cecde23f1 [sys.catalog]: This master's current role is: LEADER
14:10:08.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:08.682644 21630 sys_catalog.cc:458] T 00000000000000000000000000000000 P f34e5ae5629346b380aa3975ffecb1f3 [sys.catalog]: This master's current role is: FOLLOWER
14:10:08.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:08.693897 21642 sys_catalog.cc:455] T 00000000000000000000000000000000 P f34e5ae5629346b380aa3975ffecb1f3 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "1fd5034219e348e289cf311cecde23f1" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f34e5ae5629346b380aa3975ffecb1f3" member_type: VOTER last_known_addr { host: "127.20.207.190" port: 38271 } } peers { permanent_uuid: "1fd5034219e348e289cf311cecde23f1" member_type: VOTER last_known_addr { host: "127.20.207.189" port: 43023 } } peers { permanent_uuid: "9c6aef56265e4e5ca5ee68e82eba93c9" member_type: VOTER last_known_addr { host: "127.20.207.188" port: 42709 } } }
14:10:08.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:08.696717 21642 sys_catalog.cc:458] T 00000000000000000000000000000000 P f34e5ae5629346b380aa3975ffecb1f3 [sys.catalog]: This master's current role is: FOLLOWER
14:10:08.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:08.708267 21648 catalog_manager.cc:797] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
14:10:08.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:08.729473 21625 catalog_manager.cc:5955] T 00000000000000000000000000000000 P 1fd5034219e348e289cf311cecde23f1: Generated new TSK 0
14:10:08.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:08.730619 21625 catalog_manager.cc:1516] Initializing in-progress tserver states...
14:10:09.297 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.20.207.190:38271
14:10:09.298 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
14:10:09.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:09.299261 21310 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskBD8el9/build/tsan/bin/kudu with pid 21339
14:10:09.335 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:10:09.335 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.20.207.188:42709
14:10:09.335 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
14:10:09.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:09.336199 21310 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskBD8el9/build/tsan/bin/kudu with pid 21503
14:10:09.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:09.350705 21445 consensus_peers.cc:489] T 00000000000000000000000000000000 P 1fd5034219e348e289cf311cecde23f1 -> Peer f34e5ae5629346b380aa3975ffecb1f3 (127.20.207.190:38271): Couldn't send request to peer f34e5ae5629346b380aa3975ffecb1f3. Status: Network error: Client connection negotiation failed: client connection to 127.20.207.190:38271: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
14:10:09.370 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:10:09.370 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.20.207.189:43023
14:10:09.370 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
14:10:09.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:09.370914 21310 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskBD8el9/build/tsan/bin/kudu with pid 21419
14:10:09.393 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:10:09.393 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.20.207.190:38271
14:10:09.393 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
14:10:09.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:09.395251 21310 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskBD8el9/build/tsan/bin/kudu
14:10:09.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskBD8el9/build/tsan/bin/kudu
14:10:09.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-0/wal
14:10:09.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-0/data
14:10:09.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:10:09.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:10:09.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:10:09.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:10:09.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:10:09.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:10:09.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-0/logs
14:10:09.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-0/data/info.pb
14:10:09.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:10:09.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:10:09.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:10:09.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:10:09.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:10:09.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:10:09.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:10:09.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:10:09.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:10:09.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:10:09.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:10:09.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:10:09.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.207.190:38271
14:10:09.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.207.190
14:10:09.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37257
14:10:09.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/krb5kdc/kudu.keytab
14:10:09.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.20.207.190
14:10:09.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:10:09.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:10:09.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:10:09.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
14:10:09.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.207.148:36627
14:10:09.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:10:09.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:10:09.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:10:09.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:10:09.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.207.190:38271,127.20.207.189:43023,127.20.207.188:42709
14:10:09.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:10:09.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34701/jwks.json
14:10:09.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
14:10:09.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:10:09.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:09.726577 21652 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:10:09.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:09.727226 21652 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:10:09.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:09.728015 21652 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:10:09.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:09.768613 21652 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
14:10:09.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:09.768888 21652 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:10:09.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:09.769287 21652 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:10:09.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:09.769598 21652 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:10:09.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:09.769893 21652 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:10:09.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:09.770190 21652 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:10:09.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:09.770463 21652 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34701/jwks.json
14:10:09.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:09.819378 21652 master_runner.cc:386] Master server non-default flags:
14:10:09.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:10:09.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.207.148:36627
14:10:09.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:10:09.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:10:09.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-0/data
14:10:09.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-0/wal
14:10:09.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
14:10:09.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:10:09.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.207.190:38271,127.20.207.189:43023,127.20.207.188:42709
14:10:09.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/krb5kdc/kudu.keytab
14:10:09.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.20.207.190
14:10:09.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:10:09.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:10:09.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:10:09.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.207.190:38271
14:10:09.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:10:09.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:10:09.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:10:09.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34701/jwks.json
14:10:09.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:10:09.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:10:09.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:10:09.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:10:09.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:10:09.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-0/data/info.pb
14:10:09.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.207.190
14:10:09.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37257
14:10:09.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:10:09.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:10:09.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:10:09.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:10:09.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:10:09.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-0/logs
14:10:09.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:10:09.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:10:09.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:10:09.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:10:09.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:10:09.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 06466b25e294b4d7d455e6c56263318dc34a80ef
14:10:09.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:10:09.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 30 Aug 2025 13:43:24 UTC on 5fd53c4cbb9d
14:10:09.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7817
14:10:09.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:10:09.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:09.821038 21652 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:10:09.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:09.822108 21652 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:10:09.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:09.833568 21658 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:10:11.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:09.834784 21659 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:10:11.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:11.158785 21661 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:10:11.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:11.161302 21660 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1323 milliseconds
14:10:11.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:11.161437 21652 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:10:11.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:11.162596 21652 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:10:11.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:11.165706 21652 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:10:11.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:11.167131 21652 hybrid_clock.cc:648] HybridClock initialized: now 1756563011167087 us; error 56 us; skew 500 ppm
14:10:11.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:11.167902 21652 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:10:11.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 30 14:10:11 dist-test-slave-xkqm krb5kdc[21321](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756563011, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.207.190@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:10:11.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:11.174420 21652 init.cc:377] Logged in from keytab as kudu/127.20.207.190@KRBTEST.COM (short username kudu)
14:10:11.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:11.180366 21652 webserver.cc:480] Webserver started at http://127.20.207.190:37257/ using document root <none> and password file <none>
14:10:11.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:11.181357 21652 fs_manager.cc:362] Metadata directory not provided
14:10:11.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:11.181628 21652 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:10:11.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:11.189831 21652 fs_manager.cc:714] Time spent opening directory manager: real 0.005s user 0.004s sys 0.000s
14:10:11.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:11.194581 21670 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:10:11.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:11.195668 21652 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.001s
14:10:11.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:11.196038 21652 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-0/data,/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-0/wal
14:10:11.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f34e5ae5629346b380aa3975ffecb1f3"
14:10:11.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-30 14:10:03 on dist-test-slave-xkqm"
14:10:11.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:11.198009 21652 fs_report.cc:389] FS layout report
14:10:11.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:10:11.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-0/wal
14:10:11.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-0/wal
14:10:11.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-0/data/data
14:10:11.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:10:11.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:10:11.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:10:11.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:10:11.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:10:11.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:10:11.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:10:11.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:10:11.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:10:11.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:10:11.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:10:11.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:11.247520 21652 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:10:11.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:11.258229 21652 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:10:11.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:11.258895 21652 kserver.cc:163] Server-wide thread pool size limit: 3276
14:10:11.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:11.338666 21652 rpc_server.cc:307] RPC server started. Bound to: 127.20.207.190:38271
14:10:11.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:11.338770 21721 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.20.207.190:38271 every 8 connection(s)
14:10:11.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:11.341427 21652 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-0/data/info.pb
14:10:11.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:11.349251 21310 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskBD8el9/build/tsan/bin/kudu as pid 21652
14:10:11.349 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:10:11.350 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.20.207.188:42709
14:10:11.350 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
14:10:11.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:11.351728 21310 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskBD8el9/build/tsan/bin/kudu
14:10:11.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskBD8el9/build/tsan/bin/kudu
14:10:11.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-2/wal
14:10:11.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-2/data
14:10:11.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:10:11.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:10:11.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:10:11.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:10:11.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:10:11.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:10:11.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-2/logs
14:10:11.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-2/data/info.pb
14:10:11.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:10:11.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:10:11.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:10:11.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:10:11.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:10:11.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:10:11.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:10:11.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:10:11.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:10:11.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:10:11.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:10:11.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:10:11.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.207.188:42709
14:10:11.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.207.188
14:10:11.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38937
14:10:11.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/krb5kdc/kudu.keytab
14:10:11.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.20.207.188
14:10:11.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:10:11.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:10:11.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:10:11.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
14:10:11.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.207.148:36627
14:10:11.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:10:11.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:10:11.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:10:11.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:10:11.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.207.190:38271,127.20.207.189:43023,127.20.207.188:42709
14:10:11.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:10:11.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34701/jwks.json
14:10:11.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
14:10:11.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:10:11.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:11.353209 21722 sys_catalog.cc:263] Verifying existing consensus state
14:10:11.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:11.360719 21722 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f34e5ae5629346b380aa3975ffecb1f3: Bootstrap starting.
14:10:11.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:11.400138 21722 log.cc:826] T 00000000000000000000000000000000 P f34e5ae5629346b380aa3975ffecb1f3: Log is configured to *not* fsync() on all Append() calls
14:10:11.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:11.415288 21722 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f34e5ae5629346b380aa3975ffecb1f3: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
14:10:11.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:11.416049 21722 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f34e5ae5629346b380aa3975ffecb1f3: Bootstrap complete.
14:10:11.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:11.439105 21722 raft_consensus.cc:357] T 00000000000000000000000000000000 P f34e5ae5629346b380aa3975ffecb1f3 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f34e5ae5629346b380aa3975ffecb1f3" member_type: VOTER last_known_addr { host: "127.20.207.190" port: 38271 } } peers { permanent_uuid: "1fd5034219e348e289cf311cecde23f1" member_type: VOTER last_known_addr { host: "127.20.207.189" port: 43023 } } peers { permanent_uuid: "9c6aef56265e4e5ca5ee68e82eba93c9" member_type: VOTER last_known_addr { host: "127.20.207.188" port: 42709 } }
14:10:11.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:11.441634 21722 raft_consensus.cc:738] T 00000000000000000000000000000000 P f34e5ae5629346b380aa3975ffecb1f3 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: f34e5ae5629346b380aa3975ffecb1f3, State: Initialized, Role: FOLLOWER
14:10:11.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:11.442644 21722 consensus_queue.cc:260] T 00000000000000000000000000000000 P f34e5ae5629346b380aa3975ffecb1f3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f34e5ae5629346b380aa3975ffecb1f3" member_type: VOTER last_known_addr { host: "127.20.207.190" port: 38271 } } peers { permanent_uuid: "1fd5034219e348e289cf311cecde23f1" member_type: VOTER last_known_addr { host: "127.20.207.189" port: 43023 } } peers { permanent_uuid: "9c6aef56265e4e5ca5ee68e82eba93c9" member_type: VOTER last_known_addr { host: "127.20.207.188" port: 42709 } }
14:10:11.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:11.445623 21726 sys_catalog.cc:455] T 00000000000000000000000000000000 P f34e5ae5629346b380aa3975ffecb1f3 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f34e5ae5629346b380aa3975ffecb1f3" member_type: VOTER last_known_addr { host: "127.20.207.190" port: 38271 } } peers { permanent_uuid: "1fd5034219e348e289cf311cecde23f1" member_type: VOTER last_known_addr { host: "127.20.207.189" port: 43023 } } peers { permanent_uuid: "9c6aef56265e4e5ca5ee68e82eba93c9" member_type: VOTER last_known_addr { host: "127.20.207.188" port: 42709 } } }
14:10:11.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:11.446506 21726 sys_catalog.cc:458] T 00000000000000000000000000000000 P f34e5ae5629346b380aa3975ffecb1f3 [sys.catalog]: This master's current role is: FOLLOWER
14:10:11.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:11.447440 21722 sys_catalog.cc:564] T 00000000000000000000000000000000 P f34e5ae5629346b380aa3975ffecb1f3 [sys.catalog]: configured and running, proceeding with master startup.
14:10:11.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:11.492177 21737 catalog_manager.cc:1261] Loaded cluster ID: d49b6b05d5d64b61a603c9b5cbaf4320
14:10:11.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:11.492496 21737 catalog_manager.cc:1554] T 00000000000000000000000000000000 P f34e5ae5629346b380aa3975ffecb1f3: loading cluster ID for follower catalog manager: success
14:10:11.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 30 14:10:11 dist-test-slave-xkqm krb5kdc[21321](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756563011, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.207.190@KRBTEST.COM for kudu/127.20.207.190@KRBTEST.COM
14:10:11.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:11.501102 21737 catalog_manager.cc:1576] T 00000000000000000000000000000000 P f34e5ae5629346b380aa3975ffecb1f3: acquiring CA information for follower catalog manager: success
14:10:11.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:11.506808 21737 catalog_manager.cc:1604] T 00000000000000000000000000000000 P f34e5ae5629346b380aa3975ffecb1f3: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
14:10:11.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:11.519138 21652 master_runner.cc:186] Error getting master registration for 127.20.207.189:43023: Network error: Client connection negotiation failed: client connection to 127.20.207.189:43023: connect: Connection refused (error 111),
14:10:11.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:11.549989 21652 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:10:11.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:11.715175 21724 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:10:11.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:11.715711 21724 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:10:11.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:11.716395 21724 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:10:11.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:11.752089 21724 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
14:10:11.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:11.752439 21724 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:10:11.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:11.752887 21724 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:10:11.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:11.753204 21724 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:10:11.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:11.753518 21724 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:10:11.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:11.753814 21724 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:10:11.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:11.754052 21724 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34701/jwks.json
14:10:11.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:11.805541 21724 master_runner.cc:386] Master server non-default flags:
14:10:11.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:10:11.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.207.148:36627
14:10:11.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:10:11.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:10:11.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-2/data
14:10:11.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-2/wal
14:10:11.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
14:10:11.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:10:11.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.207.190:38271,127.20.207.189:43023,127.20.207.188:42709
14:10:11.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/krb5kdc/kudu.keytab
14:10:11.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.20.207.188
14:10:11.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:10:11.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:10:11.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:10:11.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.207.188:42709
14:10:11.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:10:11.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:10:11.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:10:11.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34701/jwks.json
14:10:11.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:10:11.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:10:11.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:10:11.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:10:11.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:10:11.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-2/data/info.pb
14:10:11.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.207.188
14:10:11.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38937
14:10:11.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:10:11.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:10:11.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:10:11.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:10:11.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:10:11.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-2/logs
14:10:11.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:10:11.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:10:11.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:10:11.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:10:11.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:10:11.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 06466b25e294b4d7d455e6c56263318dc34a80ef
14:10:11.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:10:11.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 30 Aug 2025 13:43:24 UTC on 5fd53c4cbb9d
14:10:11.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7817
14:10:11.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:10:11.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:11.807250 21724 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:10:11.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:11.808646 21724 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:10:11.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:11.821720 21746 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:10:13.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:13.049121 21752 raft_consensus.cc:491] T 00000000000000000000000000000000 P f34e5ae5629346b380aa3975ffecb1f3 [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:10:13.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:13.049728 21752 raft_consensus.cc:513] T 00000000000000000000000000000000 P f34e5ae5629346b380aa3975ffecb1f3 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f34e5ae5629346b380aa3975ffecb1f3" member_type: VOTER last_known_addr { host: "127.20.207.190" port: 38271 } } peers { permanent_uuid: "1fd5034219e348e289cf311cecde23f1" member_type: VOTER last_known_addr { host: "127.20.207.189" port: 43023 } } peers { permanent_uuid: "9c6aef56265e4e5ca5ee68e82eba93c9" member_type: VOTER last_known_addr { host: "127.20.207.188" port: 42709 } }
14:10:13.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:13.057011 21752 leader_election.cc:290] T 00000000000000000000000000000000 P f34e5ae5629346b380aa3975ffecb1f3 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 1fd5034219e348e289cf311cecde23f1 (127.20.207.189:43023), 9c6aef56265e4e5ca5ee68e82eba93c9 (127.20.207.188:42709)
14:10:13.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:13.059854 21673 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.20.207.188:42709: connect: Connection refused (error 111)
14:10:13.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:13.087112 21673 leader_election.cc:336] T 00000000000000000000000000000000 P f34e5ae5629346b380aa3975ffecb1f3 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 9c6aef56265e4e5ca5ee68e82eba93c9 (127.20.207.188:42709): Network error: Client connection negotiation failed: client connection to 127.20.207.188:42709: connect: Connection refused (error 111)
14:10:13.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:13.092109 21674 leader_election.cc:336] T 00000000000000000000000000000000 P f34e5ae5629346b380aa3975ffecb1f3 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 1fd5034219e348e289cf311cecde23f1 (127.20.207.189:43023): Network error: Client connection negotiation failed: client connection to 127.20.207.189:43023: connect: Connection refused (error 111)
14:10:13.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:13.092615 21674 leader_election.cc:304] T 00000000000000000000000000000000 P f34e5ae5629346b380aa3975ffecb1f3 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: f34e5ae5629346b380aa3975ffecb1f3; no voters: 1fd5034219e348e289cf311cecde23f1, 9c6aef56265e4e5ca5ee68e82eba93c9
14:10:13.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:13.093621 21752 raft_consensus.cc:2747] T 00000000000000000000000000000000 P f34e5ae5629346b380aa3975ffecb1f3 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
14:10:13.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:13.222967 21745 debug-util.cc:398] Leaking SignalData structure 0x7b0800037200 after lost signal to thread 21724
14:10:13.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:13.530292 21745 kernel_stack_watchdog.cc:198] Thread 21724 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 397ms:
14:10:13.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
14:10:13.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
14:10:13.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
14:10:13.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
14:10:13.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:13.531489 21724 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.708s user 0.541s sys 0.960s
14:10:13.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:13.531853 21724 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.708s user 0.541s sys 0.960s
14:10:13.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:13.532094 21748 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection timed out after 1708 milliseconds
14:10:13.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:11.823645 21747 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:10:13.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:13.533872 21749 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:10:13.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:13.534317 21724 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:10:13.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:13.538292 21724 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:10:13.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:13.541476 21724 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:10:13.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:13.542954 21724 hybrid_clock.cc:648] HybridClock initialized: now 1756563013542843 us; error 115 us; skew 500 ppm
14:10:13.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:13.543977 21724 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:10:13.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 30 14:10:13 dist-test-slave-xkqm krb5kdc[21321](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756563013, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.207.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:10:13.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:13.553288 21724 init.cc:377] Logged in from keytab as kudu/127.20.207.188@KRBTEST.COM (short username kudu)
14:10:13.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:13.559304 21724 webserver.cc:480] Webserver started at http://127.20.207.188:38937/ using document root <none> and password file <none>
14:10:13.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:13.560703 21724 fs_manager.cc:362] Metadata directory not provided
14:10:13.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:13.561033 21724 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:10:13.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:13.572424 21724 fs_manager.cc:714] Time spent opening directory manager: real 0.006s user 0.007s sys 0.000s
14:10:13.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:13.577977 21762 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:10:13.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:13.579290 21724 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
14:10:13.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:13.579707 21724 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-2/data,/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-2/wal
14:10:13.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9c6aef56265e4e5ca5ee68e82eba93c9"
14:10:13.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-30 14:10:07 on dist-test-slave-xkqm"
14:10:13.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:13.582456 21724 fs_report.cc:389] FS layout report
14:10:13.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:10:13.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-2/wal
14:10:13.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-2/wal
14:10:13.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-2/data/data
14:10:13.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:10:13.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:10:13.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:10:13.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:10:13.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:10:13.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:10:13.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:10:13.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:10:13.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:10:13.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:10:13.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:10:13.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:13.645749 21724 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:10:13.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:13.653648 21724 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:10:13.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:13.654213 21724 kserver.cc:163] Server-wide thread pool size limit: 3276
14:10:13.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:13.741717 21724 rpc_server.cc:307] RPC server started. Bound to: 127.20.207.188:42709
14:10:13.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:13.741832 21813 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.20.207.188:42709 every 8 connection(s)
14:10:13.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:13.745707 21724 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-2/data/info.pb
14:10:13.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:13.750347 21310 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskBD8el9/build/tsan/bin/kudu as pid 21724
14:10:13.751 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:10:13.751 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.20.207.189:43023
14:10:13.751 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
14:10:13.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:13.753032 21310 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskBD8el9/build/tsan/bin/kudu
14:10:13.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskBD8el9/build/tsan/bin/kudu
14:10:13.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-1/wal
14:10:13.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-1/data
14:10:13.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:10:13.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:10:13.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:10:13.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:10:13.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:10:13.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:10:13.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-1/logs
14:10:13.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-1/data/info.pb
14:10:13.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:10:13.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:10:13.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:10:13.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:10:13.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:10:13.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:10:13.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:10:13.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:10:13.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:10:13.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:10:13.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:10:13.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:10:13.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.207.189:43023
14:10:13.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.207.189
14:10:13.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=43607
14:10:13.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/krb5kdc/kudu.keytab
14:10:13.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.20.207.189
14:10:13.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:10:13.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:10:13.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:10:13.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
14:10:13.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.207.148:36627
14:10:13.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:10:13.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:10:13.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:10:13.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:10:13.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.207.190:38271,127.20.207.189:43023,127.20.207.188:42709
14:10:13.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:10:13.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34701/jwks.json
14:10:13.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
14:10:13.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:10:13.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:13.762426 21814 sys_catalog.cc:263] Verifying existing consensus state
14:10:13.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:13.771982 21814 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9c6aef56265e4e5ca5ee68e82eba93c9: Bootstrap starting.
14:10:13.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:13.835181 21814 log.cc:826] T 00000000000000000000000000000000 P 9c6aef56265e4e5ca5ee68e82eba93c9: Log is configured to *not* fsync() on all Append() calls
14:10:13.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:13.856597 21814 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9c6aef56265e4e5ca5ee68e82eba93c9: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
14:10:13.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:13.857733 21814 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9c6aef56265e4e5ca5ee68e82eba93c9: Bootstrap complete.
14:10:13.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:13.891429 21814 raft_consensus.cc:357] T 00000000000000000000000000000000 P 9c6aef56265e4e5ca5ee68e82eba93c9 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f34e5ae5629346b380aa3975ffecb1f3" member_type: VOTER last_known_addr { host: "127.20.207.190" port: 38271 } } peers { permanent_uuid: "1fd5034219e348e289cf311cecde23f1" member_type: VOTER last_known_addr { host: "127.20.207.189" port: 43023 } } peers { permanent_uuid: "9c6aef56265e4e5ca5ee68e82eba93c9" member_type: VOTER last_known_addr { host: "127.20.207.188" port: 42709 } }
14:10:13.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:13.894696 21814 raft_consensus.cc:738] T 00000000000000000000000000000000 P 9c6aef56265e4e5ca5ee68e82eba93c9 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9c6aef56265e4e5ca5ee68e82eba93c9, State: Initialized, Role: FOLLOWER
14:10:13.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:13.895678 21814 consensus_queue.cc:260] T 00000000000000000000000000000000 P 9c6aef56265e4e5ca5ee68e82eba93c9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f34e5ae5629346b380aa3975ffecb1f3" member_type: VOTER last_known_addr { host: "127.20.207.190" port: 38271 } } peers { permanent_uuid: "1fd5034219e348e289cf311cecde23f1" member_type: VOTER last_known_addr { host: "127.20.207.189" port: 43023 } } peers { permanent_uuid: "9c6aef56265e4e5ca5ee68e82eba93c9" member_type: VOTER last_known_addr { host: "127.20.207.188" port: 42709 } }
14:10:13.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:13.898690 21818 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9c6aef56265e4e5ca5ee68e82eba93c9 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f34e5ae5629346b380aa3975ffecb1f3" member_type: VOTER last_known_addr { host: "127.20.207.190" port: 38271 } } peers { permanent_uuid: "1fd5034219e348e289cf311cecde23f1" member_type: VOTER last_known_addr { host: "127.20.207.189" port: 43023 } } peers { permanent_uuid: "9c6aef56265e4e5ca5ee68e82eba93c9" member_type: VOTER last_known_addr { host: "127.20.207.188" port: 42709 } } }
14:10:13.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:13.899935 21818 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9c6aef56265e4e5ca5ee68e82eba93c9 [sys.catalog]: This master's current role is: FOLLOWER
14:10:13.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:13.908744 21814 sys_catalog.cc:564] T 00000000000000000000000000000000 P 9c6aef56265e4e5ca5ee68e82eba93c9 [sys.catalog]: configured and running, proceeding with master startup.
14:10:14.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:14.018364 21295 debug-util.cc:398] Leaking SignalData structure 0x7b08000ac480 after lost signal to thread 21170
14:10:14.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:14.019618 21295 debug-util.cc:398] Leaking SignalData structure 0x7b08000b3900 after lost signal to thread 21298
14:10:14.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:14.207821 21829 catalog_manager.cc:1261] Loaded cluster ID: d49b6b05d5d64b61a603c9b5cbaf4320
14:10:14.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 30 14:10:14 dist-test-slave-xkqm krb5kdc[21321](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756563013, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.207.188@KRBTEST.COM for kudu/127.20.207.190@KRBTEST.COM
14:10:14.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:14.208101 21829 catalog_manager.cc:1554] T 00000000000000000000000000000000 P 9c6aef56265e4e5ca5ee68e82eba93c9: loading cluster ID for follower catalog manager: success
14:10:14.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:14.258241 21829 catalog_manager.cc:1576] T 00000000000000000000000000000000 P 9c6aef56265e4e5ca5ee68e82eba93c9: acquiring CA information for follower catalog manager: success
14:10:14.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:14.275857 21829 catalog_manager.cc:1604] T 00000000000000000000000000000000 P 9c6aef56265e4e5ca5ee68e82eba93c9: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
14:10:14.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:14.274598 21724 master_runner.cc:186] Error getting master registration for 127.20.207.189:43023: Network error: Client connection negotiation failed: client connection to 127.20.207.189:43023: connect: Connection refused (error 111),
14:10:14.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:14.319170 21724 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:10:14.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:14.425345 21816 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:10:14.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:14.426872 21816 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:10:14.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:14.428253 21816 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:10:14.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:14.497270 21816 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
14:10:14.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:14.497876 21816 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:10:14.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:14.498574 21816 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:10:14.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:14.499251 21816 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:10:14.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:14.499725 21816 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:10:14.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:14.500160 21816 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:10:14.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:14.500558 21816 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34701/jwks.json
14:10:14.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:14.573232 21816 master_runner.cc:386] Master server non-default flags:
14:10:14.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:10:14.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.207.148:36627
14:10:14.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:10:14.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:10:14.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-1/data
14:10:14.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-1/wal
14:10:14.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
14:10:14.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:10:14.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.207.190:38271,127.20.207.189:43023,127.20.207.188:42709
14:10:14.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/krb5kdc/kudu.keytab
14:10:14.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.20.207.189
14:10:14.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:10:14.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:10:14.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:10:14.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.207.189:43023
14:10:14.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:10:14.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:10:14.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:10:14.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34701/jwks.json
14:10:14.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:10:14.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:10:14.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:10:14.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:10:14.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:10:14.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-1/data/info.pb
14:10:14.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.207.189
14:10:14.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=43607
14:10:14.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:10:14.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:10:14.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:10:14.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:10:14.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:10:14.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-1/logs
14:10:14.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:10:14.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:10:14.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:10:14.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:10:14.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:10:14.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 06466b25e294b4d7d455e6c56263318dc34a80ef
14:10:14.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:10:14.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 30 Aug 2025 13:43:24 UTC on 5fd53c4cbb9d
14:10:14.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7817
14:10:14.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:10:14.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:14.576761 21816 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:10:14.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:14.578395 21816 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:10:14.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:14.621321 21841 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:10:14.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:14.719707 21848 raft_consensus.cc:491] T 00000000000000000000000000000000 P f34e5ae5629346b380aa3975ffecb1f3 [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:10:14.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:14.721472 21848 raft_consensus.cc:513] T 00000000000000000000000000000000 P f34e5ae5629346b380aa3975ffecb1f3 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f34e5ae5629346b380aa3975ffecb1f3" member_type: VOTER last_known_addr { host: "127.20.207.190" port: 38271 } } peers { permanent_uuid: "1fd5034219e348e289cf311cecde23f1" member_type: VOTER last_known_addr { host: "127.20.207.189" port: 43023 } } peers { permanent_uuid: "9c6aef56265e4e5ca5ee68e82eba93c9" member_type: VOTER last_known_addr { host: "127.20.207.188" port: 42709 } }
14:10:14.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:14.744334 21848 leader_election.cc:290] T 00000000000000000000000000000000 P f34e5ae5629346b380aa3975ffecb1f3 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 1fd5034219e348e289cf311cecde23f1 (127.20.207.189:43023), 9c6aef56265e4e5ca5ee68e82eba93c9 (127.20.207.188:42709)
14:10:14.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:14.782763 21674 leader_election.cc:336] T 00000000000000000000000000000000 P f34e5ae5629346b380aa3975ffecb1f3 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 1fd5034219e348e289cf311cecde23f1 (127.20.207.189:43023): Network error: Client connection negotiation failed: client connection to 127.20.207.189:43023: connect: Connection refused (error 111)
14:10:14.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:14.809947 21789 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "f34e5ae5629346b380aa3975ffecb1f3" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "9c6aef56265e4e5ca5ee68e82eba93c9" is_pre_election: true
14:10:14.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:14.810972 21789 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 9c6aef56265e4e5ca5ee68e82eba93c9 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate f34e5ae5629346b380aa3975ffecb1f3 in term 1.
14:10:14.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:14.812858 21673 leader_election.cc:304] T 00000000000000000000000000000000 P f34e5ae5629346b380aa3975ffecb1f3 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 9c6aef56265e4e5ca5ee68e82eba93c9, f34e5ae5629346b380aa3975ffecb1f3; no voters: 1fd5034219e348e289cf311cecde23f1
14:10:14.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:14.813758 21848 raft_consensus.cc:2802] T 00000000000000000000000000000000 P f34e5ae5629346b380aa3975ffecb1f3 [term 1 FOLLOWER]: Leader pre-election won for term 2
14:10:14.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:14.814096 21848 raft_consensus.cc:491] T 00000000000000000000000000000000 P f34e5ae5629346b380aa3975ffecb1f3 [term 1 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
14:10:14.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:14.814414 21848 raft_consensus.cc:3058] T 00000000000000000000000000000000 P f34e5ae5629346b380aa3975ffecb1f3 [term 1 FOLLOWER]: Advancing to term 2
14:10:14.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:14.837628 21848 raft_consensus.cc:513] T 00000000000000000000000000000000 P f34e5ae5629346b380aa3975ffecb1f3 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f34e5ae5629346b380aa3975ffecb1f3" member_type: VOTER last_known_addr { host: "127.20.207.190" port: 38271 } } peers { permanent_uuid: "1fd5034219e348e289cf311cecde23f1" member_type: VOTER last_known_addr { host: "127.20.207.189" port: 43023 } } peers { permanent_uuid: "9c6aef56265e4e5ca5ee68e82eba93c9" member_type: VOTER last_known_addr { host: "127.20.207.188" port: 42709 } }
14:10:14.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:14.846567 21674 leader_election.cc:336] T 00000000000000000000000000000000 P f34e5ae5629346b380aa3975ffecb1f3 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 1fd5034219e348e289cf311cecde23f1 (127.20.207.189:43023): Network error: Client connection negotiation failed: client connection to 127.20.207.189:43023: connect: Connection refused (error 111)
14:10:14.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:14.853087 21789 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "f34e5ae5629346b380aa3975ffecb1f3" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "9c6aef56265e4e5ca5ee68e82eba93c9"
14:10:14.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:14.853967 21789 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 9c6aef56265e4e5ca5ee68e82eba93c9 [term 1 FOLLOWER]: Advancing to term 2
14:10:14.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:14.867326 21848 leader_election.cc:290] T 00000000000000000000000000000000 P f34e5ae5629346b380aa3975ffecb1f3 [CANDIDATE]: Term 2 election: Requested vote from peers 1fd5034219e348e289cf311cecde23f1 (127.20.207.189:43023), 9c6aef56265e4e5ca5ee68e82eba93c9 (127.20.207.188:42709)
14:10:14.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:14.874563 21789 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 9c6aef56265e4e5ca5ee68e82eba93c9 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate f34e5ae5629346b380aa3975ffecb1f3 in term 2.
14:10:14.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:14.878145 21673 leader_election.cc:304] T 00000000000000000000000000000000 P f34e5ae5629346b380aa3975ffecb1f3 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 9c6aef56265e4e5ca5ee68e82eba93c9, f34e5ae5629346b380aa3975ffecb1f3; no voters: 1fd5034219e348e289cf311cecde23f1
14:10:14.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:14.879417 21848 raft_consensus.cc:2802] T 00000000000000000000000000000000 P f34e5ae5629346b380aa3975ffecb1f3 [term 2 FOLLOWER]: Leader election won for term 2
14:10:14.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:14.900739 21848 raft_consensus.cc:695] T 00000000000000000000000000000000 P f34e5ae5629346b380aa3975ffecb1f3 [term 2 LEADER]: Becoming Leader. State: Replica: f34e5ae5629346b380aa3975ffecb1f3, State: Running, Role: LEADER
14:10:14.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:14.902851 21848 consensus_queue.cc:237] T 00000000000000000000000000000000 P f34e5ae5629346b380aa3975ffecb1f3 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f34e5ae5629346b380aa3975ffecb1f3" member_type: VOTER last_known_addr { host: "127.20.207.190" port: 38271 } } peers { permanent_uuid: "1fd5034219e348e289cf311cecde23f1" member_type: VOTER last_known_addr { host: "127.20.207.189" port: 43023 } } peers { permanent_uuid: "9c6aef56265e4e5ca5ee68e82eba93c9" member_type: VOTER last_known_addr { host: "127.20.207.188" port: 42709 } }
14:10:14.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:14.933395 21853 sys_catalog.cc:455] T 00000000000000000000000000000000 P f34e5ae5629346b380aa3975ffecb1f3 [sys.catalog]: SysCatalogTable state changed. Reason: New leader f34e5ae5629346b380aa3975ffecb1f3. Latest consensus state: current_term: 2 leader_uuid: "f34e5ae5629346b380aa3975ffecb1f3" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f34e5ae5629346b380aa3975ffecb1f3" member_type: VOTER last_known_addr { host: "127.20.207.190" port: 38271 } } peers { permanent_uuid: "1fd5034219e348e289cf311cecde23f1" member_type: VOTER last_known_addr { host: "127.20.207.189" port: 43023 } } peers { permanent_uuid: "9c6aef56265e4e5ca5ee68e82eba93c9" member_type: VOTER last_known_addr { host: "127.20.207.188" port: 42709 } } }
14:10:14.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:14.934520 21853 sys_catalog.cc:458] T 00000000000000000000000000000000 P f34e5ae5629346b380aa3975ffecb1f3 [sys.catalog]: This master's current role is: LEADER
14:10:14.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:14.949615 21855 catalog_manager.cc:1477] Loading table and tablet metadata into memory...
14:10:14.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:14.966495 21855 catalog_manager.cc:1486] Initializing Kudu cluster ID...
14:10:14.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:14.974093 21855 catalog_manager.cc:1261] Loaded cluster ID: d49b6b05d5d64b61a603c9b5cbaf4320
14:10:14.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:14.974429 21855 catalog_manager.cc:1497] Initializing Kudu internal certificate authority...
14:10:14.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:14.979120 21855 catalog_manager.cc:1506] Loading token signing keys...
14:10:14.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:14.983718 21855 catalog_manager.cc:5966] T 00000000000000000000000000000000 P f34e5ae5629346b380aa3975ffecb1f3: Loaded TSK: 0
14:10:15.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:15.002456 21855 catalog_manager.cc:1516] Initializing in-progress tserver states...
14:10:15.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:15.033028 21154 debug-util.cc:398] Leaking SignalData structure 0x7b08000b53e0 after lost signal to thread 21026
14:10:15.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:15.035384 21154 debug-util.cc:398] Leaking SignalData structure 0x7b08000b54a0 after lost signal to thread 21157
14:10:15.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:15.291107 20734 debug-util.cc:398] Leaking SignalData structure 0x7b080006f100 after lost signal to thread 20669
14:10:15.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:15.292284 20734 debug-util.cc:398] Leaking SignalData structure 0x7b080007c880 after lost signal to thread 20737
14:10:15.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:15.315661 21789 raft_consensus.cc:1273] T 00000000000000000000000000000000 P 9c6aef56265e4e5ca5ee68e82eba93c9 [term 2 FOLLOWER]: Refusing update from remote peer f34e5ae5629346b380aa3975ffecb1f3: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 2 index: 5. (index mismatch)
14:10:15.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:15.317711 21853 consensus_queue.cc:1035] T 00000000000000000000000000000000 P f34e5ae5629346b380aa3975ffecb1f3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9c6aef56265e4e5ca5ee68e82eba93c9" member_type: VOTER last_known_addr { host: "127.20.207.188" port: 42709 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
14:10:15.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:15.372761 21856 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9c6aef56265e4e5ca5ee68e82eba93c9 [sys.catalog]: SysCatalogTable state changed. Reason: New leader f34e5ae5629346b380aa3975ffecb1f3. Latest consensus state: current_term: 2 leader_uuid: "f34e5ae5629346b380aa3975ffecb1f3" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f34e5ae5629346b380aa3975ffecb1f3" member_type: VOTER last_known_addr { host: "127.20.207.190" port: 38271 } } peers { permanent_uuid: "1fd5034219e348e289cf311cecde23f1" member_type: VOTER last_known_addr { host: "127.20.207.189" port: 43023 } } peers { permanent_uuid: "9c6aef56265e4e5ca5ee68e82eba93c9" member_type: VOTER last_known_addr { host: "127.20.207.188" port: 42709 } } }
14:10:15.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:15.383528 21853 sys_catalog.cc:455] T 00000000000000000000000000000000 P f34e5ae5629346b380aa3975ffecb1f3 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "f34e5ae5629346b380aa3975ffecb1f3" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f34e5ae5629346b380aa3975ffecb1f3" member_type: VOTER last_known_addr { host: "127.20.207.190" port: 38271 } } peers { permanent_uuid: "1fd5034219e348e289cf311cecde23f1" member_type: VOTER last_known_addr { host: "127.20.207.189" port: 43023 } } peers { permanent_uuid: "9c6aef56265e4e5ca5ee68e82eba93c9" member_type: VOTER last_known_addr { host: "127.20.207.188" port: 42709 } } }
14:10:15.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:15.385792 21853 sys_catalog.cc:458] T 00000000000000000000000000000000 P f34e5ae5629346b380aa3975ffecb1f3 [sys.catalog]: This master's current role is: LEADER
14:10:15.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:15.392612 21856 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9c6aef56265e4e5ca5ee68e82eba93c9 [sys.catalog]: This master's current role is: FOLLOWER
14:10:15.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:15.418382 21674 consensus_peers.cc:489] T 00000000000000000000000000000000 P f34e5ae5629346b380aa3975ffecb1f3 -> Peer 1fd5034219e348e289cf311cecde23f1 (127.20.207.189:43023): Couldn't send request to peer 1fd5034219e348e289cf311cecde23f1. Status: Network error: Client connection negotiation failed: client connection to 127.20.207.189:43023: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
14:10:15.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:15.424448 21856 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9c6aef56265e4e5ca5ee68e82eba93c9 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "f34e5ae5629346b380aa3975ffecb1f3" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f34e5ae5629346b380aa3975ffecb1f3" member_type: VOTER last_known_addr { host: "127.20.207.190" port: 38271 } } peers { permanent_uuid: "1fd5034219e348e289cf311cecde23f1" member_type: VOTER last_known_addr { host: "127.20.207.189" port: 43023 } } peers { permanent_uuid: "9c6aef56265e4e5ca5ee68e82eba93c9" member_type: VOTER last_known_addr { host: "127.20.207.188" port: 42709 } } }
14:10:15.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:15.425288 21856 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9c6aef56265e4e5ca5ee68e82eba93c9 [sys.catalog]: This master's current role is: FOLLOWER
14:10:16.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:16.425981 20813 debug-util.cc:398] Leaking SignalData structure 0x7b08000b6960 after lost signal to thread 20745
14:10:16.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:16.427001 20813 debug-util.cc:398] Leaking SignalData structure 0x7b080009e000 after lost signal to thread 20816
14:10:16.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:16.583797 21652 master_runner.cc:186] Error getting master registration for 127.20.207.189:43023: Network error: Client connection negotiation failed: client connection to 127.20.207.189:43023: connect: Connection refused (error 111),
14:10:16.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:16.584326 21652 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:10:16.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:16.670269 20659 debug-util.cc:398] Leaking SignalData structure 0x7b0800073060 after lost signal to thread 20596
14:10:16.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:16.671281 20659 debug-util.cc:398] Leaking SignalData structure 0x7b080009b1e0 after lost signal to thread 20662
14:10:18.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:18.006743 21674 consensus_peers.cc:489] T 00000000000000000000000000000000 P f34e5ae5629346b380aa3975ffecb1f3 -> Peer 1fd5034219e348e289cf311cecde23f1 (127.20.207.189:43023): Couldn't send request to peer 1fd5034219e348e289cf311cecde23f1. Status: Network error: Client connection negotiation failed: client connection to 127.20.207.189:43023: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
14:10:18.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:18.384738 21674 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.20.207.189:43023: connect: Connection refused (error 111) [suppressed 9 similar messages]
14:10:19.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:19.351810 21724 master_runner.cc:186] Error getting master registration for 127.20.207.189:43023: Network error: Client connection negotiation failed: client connection to 127.20.207.189:43023: connect: Connection refused (error 111),
14:10:19.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:19.352265 21724 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:10:19.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:14.628528 21842 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:10:19.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:19.448797 21816 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 4.821s user 0.657s sys 1.069s
14:10:19.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:16.010752 21840 debug-util.cc:398] Leaking SignalData structure 0x7b0800037200 after lost signal to thread 21816
14:10:19.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:19.450789 21845 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:10:19.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:19.451967 21816 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 4.824s user 0.658s sys 1.069s
14:10:19.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:19.453037 21816 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:10:19.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:19.451397 21843 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 4824 milliseconds
14:10:19.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:19.458796 21816 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:10:19.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:19.484519 21816 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:10:19.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:19.486059 21816 hybrid_clock.cc:648] HybridClock initialized: now 1756563019486116 us; error 173 us; skew 500 ppm
14:10:19.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:19.487186 21816 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:10:19.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 30 14:10:19 dist-test-slave-xkqm krb5kdc[21321](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756563019, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.207.189@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:10:19.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:19.495292 21816 init.cc:377] Logged in from keytab as kudu/127.20.207.189@KRBTEST.COM (short username kudu)
14:10:19.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:19.528158 21816 webserver.cc:480] Webserver started at http://127.20.207.189:43607/ using document root <none> and password file <none>
14:10:19.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:19.529532 21816 fs_manager.cc:362] Metadata directory not provided
14:10:19.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:19.529855 21816 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:10:19.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:19.550824 21816 fs_manager.cc:714] Time spent opening directory manager: real 0.016s user 0.005s sys 0.000s
14:10:19.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:19.567605 21883 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:10:19.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:19.569096 21816 fs_manager.cc:730] Time spent opening block manager: real 0.015s user 0.002s sys 0.004s
14:10:19.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:19.569538 21816 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-1/data,/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-1/wal
14:10:19.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1fd5034219e348e289cf311cecde23f1"
14:10:19.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-30 14:10:05 on dist-test-slave-xkqm"
14:10:19.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:19.572491 21816 fs_report.cc:389] FS layout report
14:10:19.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:10:19.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-1/wal
14:10:19.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-1/wal
14:10:19.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-1/data/data
14:10:19.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:10:19.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:10:19.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:10:19.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:10:19.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:10:19.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:10:19.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:10:19.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:10:19.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:10:19.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:10:19.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:10:19.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:19.743947 21816 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:10:19.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:19.783681 21816 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:10:19.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:19.784456 21816 kserver.cc:163] Server-wide thread pool size limit: 3276
14:10:20.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:20.127852 21011 debug-util.cc:398] Leaking SignalData structure 0x7b08000b8d60 after lost signal to thread 20884
14:10:20.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:20.129060 21011 debug-util.cc:398] Leaking SignalData structure 0x7b08000ac960 after lost signal to thread 21014
14:10:20.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:20.263765 21935 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.20.207.189:43023 every 8 connection(s)
14:10:20.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:20.264434 21816 rpc_server.cc:307] RPC server started. Bound to: 127.20.207.189:43023
14:10:20.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:20.268729 21816 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-1/data/info.pb
14:10:20.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:20.277895 21310 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskBD8el9/build/tsan/bin/kudu as pid 21816
14:10:20.279 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:10:20.293 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.20.207.190:38271(127.20.207.190:38271)] connecting to peer
14:10:20.301 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.20.207.188:42709(127.20.207.188:42709)] connecting to peer
14:10:20.312 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.20.207.189:43023(127.20.207.189:43023)] connecting to peer
14:10:20.324 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.20.207.190:38271(127.20.207.190:38271)] Successfully connected to peer
14:10:20.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:20.316042 21936 sys_catalog.cc:263] Verifying existing consensus state
14:10:20.328 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.20.207.189:43023(127.20.207.189:43023)] Successfully connected to peer
14:10:20.330 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.20.207.188:42709(127.20.207.188:42709)] Successfully connected to peer
14:10:20.353 [DEBUG - kudu-nio-0] (SecurityContext.java:222) Refreshing Kerberos credentials...
14:10:20.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:20.354698 21936 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1fd5034219e348e289cf311cecde23f1: Bootstrap starting.
14:10:20.355 [DEBUG - kudu-nio-0] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/krb5kdc/krb5cc
14:10:20.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 30 14:10:20 dist-test-slave-xkqm krb5kdc[21321](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756563008, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:10:20.365 [DEBUG - kudu-nio-0] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
14:10:20.365 [INFO - kudu-nio-0] (SecurityContext.java:256) Successfully refreshed Kerberos credentials from ticket cache
14:10:20.366 [DEBUG - kudu-nio-1] (SecurityContext.java:222) Refreshing Kerberos credentials...
14:10:20.367 [DEBUG - kudu-nio-1] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/krb5kdc/krb5cc
14:10:20.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 30 14:10:20 dist-test-slave-xkqm krb5kdc[21321](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756563008, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:10:20.372 [DEBUG - kudu-nio-1] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
14:10:20.380 [INFO - kudu-nio-1] (SecurityContext.java:256) Successfully refreshed Kerberos credentials from ticket cache
14:10:20.381 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.20.207.188
14:10:20.389 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.20.207.190
14:10:20.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 30 14:10:20 dist-test-slave-xkqm krb5kdc[21321](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756563008, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.20.207.188@KRBTEST.COM
14:10:20.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 30 14:10:20 dist-test-slave-xkqm krb5kdc[21321](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756563008, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.20.207.190@KRBTEST.COM
14:10:20.468 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x84ad6711, L:/127.0.0.1:55154 - R:/127.20.207.188:42709] using SASL/GSSAPI
14:10:20.472 [DEBUG - kudu-nio-0] (SecurityContext.java:222) Refreshing Kerberos credentials...
14:10:20.472 [DEBUG - kudu-nio-0] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/krb5kdc/krb5cc
14:10:20.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 30 14:10:20 dist-test-slave-xkqm krb5kdc[21321](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756563008, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:10:20.489 [DEBUG - kudu-nio-0] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
14:10:20.489 [INFO - kudu-nio-0] (SecurityContext.java:256) Successfully refreshed Kerberos credentials from ticket cache
14:10:20.490 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.20.207.189
14:10:20.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 30 14:10:20 dist-test-slave-xkqm krb5kdc[21321](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756563011, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.207.190@KRBTEST.COM for kudu/127.20.207.189@KRBTEST.COM
14:10:20.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:20.530766 21936 log.cc:826] T 00000000000000000000000000000000 P 1fd5034219e348e289cf311cecde23f1: Log is configured to *not* fsync() on all Append() calls
14:10:20.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 30 14:10:20 dist-test-slave-xkqm krb5kdc[21321](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756563008, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.20.207.189@KRBTEST.COM
14:10:20.542 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x7ef86f10, L:/127.0.0.1:41818 - R:/127.20.207.190:38271] using SASL/GSSAPI
14:10:20.551 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xecd8ba6b, L:/127.0.0.1:38848 - R:/127.20.207.189:43023] using SASL/GSSAPI
14:10:20.571 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
14:10:20.572 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=ListTabletServers, tablet=null, attempt=1, TimeoutTracker(timeout=30000, elapsed=293), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.207.190:38271, [8ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.207.188:42709, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.207.189:43023, [192ms] Sub RPC ConnectToMaster: received response from server master-127.20.207.188:42709: OK, [260ms] Sub RPC ConnectToMaster: received response from server master-127.20.207.190:38271: OK, deferred=null) after lookup
14:10:20.581 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.20.207.189:43023: Catalog manager is not initialized. State: Starting
14:10:20.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:20.650696 21936 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1fd5034219e348e289cf311cecde23f1: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
14:10:20.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:20.651775 21936 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1fd5034219e348e289cf311cecde23f1: Bootstrap complete.
14:10:20.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:20.785497 21936 raft_consensus.cc:357] T 00000000000000000000000000000000 P 1fd5034219e348e289cf311cecde23f1 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f34e5ae5629346b380aa3975ffecb1f3" member_type: VOTER last_known_addr { host: "127.20.207.190" port: 38271 } } peers { permanent_uuid: "1fd5034219e348e289cf311cecde23f1" member_type: VOTER last_known_addr { host: "127.20.207.189" port: 43023 } } peers { permanent_uuid: "9c6aef56265e4e5ca5ee68e82eba93c9" member_type: VOTER last_known_addr { host: "127.20.207.188" port: 42709 } }
14:10:20.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:20.789242 21936 raft_consensus.cc:738] T 00000000000000000000000000000000 P 1fd5034219e348e289cf311cecde23f1 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1fd5034219e348e289cf311cecde23f1, State: Initialized, Role: FOLLOWER
14:10:20.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:20.790473 21936 consensus_queue.cc:260] T 00000000000000000000000000000000 P 1fd5034219e348e289cf311cecde23f1 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f34e5ae5629346b380aa3975ffecb1f3" member_type: VOTER last_known_addr { host: "127.20.207.190" port: 38271 } } peers { permanent_uuid: "1fd5034219e348e289cf311cecde23f1" member_type: VOTER last_known_addr { host: "127.20.207.189" port: 43023 } } peers { permanent_uuid: "9c6aef56265e4e5ca5ee68e82eba93c9" member_type: VOTER last_known_addr { host: "127.20.207.188" port: 42709 } }
14:10:20.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:20.812323 21947 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1fd5034219e348e289cf311cecde23f1 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f34e5ae5629346b380aa3975ffecb1f3" member_type: VOTER last_known_addr { host: "127.20.207.190" port: 38271 } } peers { permanent_uuid: "1fd5034219e348e289cf311cecde23f1" member_type: VOTER last_known_addr { host: "127.20.207.189" port: 43023 } } peers { permanent_uuid: "9c6aef56265e4e5ca5ee68e82eba93c9" member_type: VOTER last_known_addr { host: "127.20.207.188" port: 42709 } } }
14:10:20.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:20.813212 21947 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1fd5034219e348e289cf311cecde23f1 [sys.catalog]: This master's current role is: FOLLOWER
14:10:20.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:20.822383 21936 sys_catalog.cc:564] T 00000000000000000000000000000000 P 1fd5034219e348e289cf311cecde23f1 [sys.catalog]: configured and running, proceeding with master startup.
14:10:20.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:20.824055 21911 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 1fd5034219e348e289cf311cecde23f1 [term 1 FOLLOWER]: Advancing to term 2
14:10:20.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:20.882474 21947 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1fd5034219e348e289cf311cecde23f1 [sys.catalog]: SysCatalogTable state changed. Reason: New leader f34e5ae5629346b380aa3975ffecb1f3. Latest consensus state: current_term: 2 leader_uuid: "f34e5ae5629346b380aa3975ffecb1f3" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f34e5ae5629346b380aa3975ffecb1f3" member_type: VOTER last_known_addr { host: "127.20.207.190" port: 38271 } } peers { permanent_uuid: "1fd5034219e348e289cf311cecde23f1" member_type: VOTER last_known_addr { host: "127.20.207.189" port: 43023 } } peers { permanent_uuid: "9c6aef56265e4e5ca5ee68e82eba93c9" member_type: VOTER last_known_addr { host: "127.20.207.188" port: 42709 } } }
14:10:20.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:20.883371 21947 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1fd5034219e348e289cf311cecde23f1 [sys.catalog]: This master's current role is: FOLLOWER
14:10:20.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:20.886337 21954 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1fd5034219e348e289cf311cecde23f1 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "f34e5ae5629346b380aa3975ffecb1f3" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f34e5ae5629346b380aa3975ffecb1f3" member_type: VOTER last_known_addr { host: "127.20.207.190" port: 38271 } } peers { permanent_uuid: "1fd5034219e348e289cf311cecde23f1" member_type: VOTER last_known_addr { host: "127.20.207.189" port: 43023 } } peers { permanent_uuid: "9c6aef56265e4e5ca5ee68e82eba93c9" member_type: VOTER last_known_addr { host: "127.20.207.188" port: 42709 } } }
14:10:20.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:20.887193 21954 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1fd5034219e348e289cf311cecde23f1 [sys.catalog]: This master's current role is: FOLLOWER
14:10:21.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 30 14:10:21 dist-test-slave-xkqm krb5kdc[21321](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756563019, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.207.189@KRBTEST.COM for kudu/127.20.207.190@KRBTEST.COM
14:10:21.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:21.023649 21961 catalog_manager.cc:1261] Loaded cluster ID: d49b6b05d5d64b61a603c9b5cbaf4320
14:10:21.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:21.023927 21961 catalog_manager.cc:1554] T 00000000000000000000000000000000 P 1fd5034219e348e289cf311cecde23f1: loading cluster ID for follower catalog manager: success
14:10:21.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:21.037474 21961 catalog_manager.cc:1576] T 00000000000000000000000000000000 P 1fd5034219e348e289cf311cecde23f1: acquiring CA information for follower catalog manager: success
14:10:21.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:21.059578 21961 catalog_manager.cc:1604] T 00000000000000000000000000000000 P 1fd5034219e348e289cf311cecde23f1: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
14:10:21.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:21.070024 21964 sys_catalog.cc:455] T 00000000000000000000000000000000 P f34e5ae5629346b380aa3975ffecb1f3 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "f34e5ae5629346b380aa3975ffecb1f3" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f34e5ae5629346b380aa3975ffecb1f3" member_type: VOTER last_known_addr { host: "127.20.207.190" port: 38271 } } peers { permanent_uuid: "1fd5034219e348e289cf311cecde23f1" member_type: VOTER last_known_addr { host: "127.20.207.189" port: 43023 } } peers { permanent_uuid: "9c6aef56265e4e5ca5ee68e82eba93c9" member_type: VOTER last_known_addr { host: "127.20.207.188" port: 42709 } } }
14:10:21.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:21.073009 21964 sys_catalog.cc:458] T 00000000000000000000000000000000 P f34e5ae5629346b380aa3975ffecb1f3 [sys.catalog]: This master's current role is: LEADER
14:10:21.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 30 14:10:21 dist-test-slave-xkqm krb5kdc[21321](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756563021, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.207.190@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:10:21.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:21.179212 21666 init.cc:295] Successfully reacquired a new kerberos TGT
14:10:21.596 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.20.207.190:38271
14:10:21.599 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
14:10:21.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:21.599356 21310 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskBD8el9/build/tsan/bin/kudu with pid 21652
14:10:21.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:21.679642 21763 connection.cc:537] server connection from 127.0.0.1:47498 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
14:10:21.681 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.20.207.190:38271(127.20.207.190:38271)] handling channelInactive
14:10:21.681 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.20.207.190:38271(127.20.207.190:38271)] cleaning up while in state READY due to: connection closed
14:10:21.684 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:10:21.685 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.20.207.188:42709
14:10:21.685 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
14:10:21.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:21.685801 21310 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskBD8el9/build/tsan/bin/kudu with pid 21724
14:10:21.754 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.20.207.188:42709(127.20.207.188:42709)] handling channelInactive
14:10:21.754 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.20.207.188:42709(127.20.207.188:42709)] cleaning up while in state READY due to: connection closed
14:10:21.777 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:10:21.777 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.20.207.189:43023
14:10:21.778 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
14:10:21.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:21.778403 21310 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskBD8el9/build/tsan/bin/kudu with pid 21816
14:10:21.812 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.20.207.189:43023(127.20.207.189:43023)] handling channelInactive
14:10:21.812 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.20.207.189:43023(127.20.207.189:43023)] cleaning up while in state READY due to: connection closed
14:10:21.824 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:10:21.826 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.20.207.190:38271
14:10:21.827 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
14:10:21.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:21.833874 21310 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskBD8el9/build/tsan/bin/kudu
14:10:21.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskBD8el9/build/tsan/bin/kudu
14:10:21.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-0/wal
14:10:21.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-0/data
14:10:21.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:10:21.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:10:21.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:10:21.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:10:21.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:10:21.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:10:21.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-0/logs
14:10:21.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-0/data/info.pb
14:10:21.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:10:21.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:10:21.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:10:21.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:10:21.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:10:21.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:10:21.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:10:21.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:10:21.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:10:21.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:10:21.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:10:21.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:10:21.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.207.190:38271
14:10:21.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.207.190
14:10:21.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37257
14:10:21.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/krb5kdc/kudu.keytab
14:10:21.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.20.207.190
14:10:21.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:10:21.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:10:21.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:10:21.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
14:10:21.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.207.148:36627
14:10:21.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:10:21.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:10:21.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:10:21.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:10:21.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.207.190:38271,127.20.207.189:43023,127.20.207.188:42709
14:10:21.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:10:21.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34701/jwks.json
14:10:21.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
14:10:21.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:10:22.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:22.227536 21968 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:10:22.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:22.228091 21968 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:10:22.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:22.228783 21968 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:10:22.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:22.263231 21968 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
14:10:22.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:22.263468 21968 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:10:22.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:22.263748 21968 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:10:22.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:22.263947 21968 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:10:22.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:22.264142 21968 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:10:22.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:22.264394 21968 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:10:22.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:22.264568 21968 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34701/jwks.json
14:10:22.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:22.317996 21968 master_runner.cc:386] Master server non-default flags:
14:10:22.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:10:22.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.207.148:36627
14:10:22.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:10:22.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:10:22.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-0/data
14:10:22.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-0/wal
14:10:22.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
14:10:22.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:10:22.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.207.190:38271,127.20.207.189:43023,127.20.207.188:42709
14:10:22.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/krb5kdc/kudu.keytab
14:10:22.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.20.207.190
14:10:22.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:10:22.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:10:22.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:10:22.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.207.190:38271
14:10:22.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:10:22.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:10:22.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:10:22.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34701/jwks.json
14:10:22.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:10:22.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:10:22.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:10:22.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:10:22.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:10:22.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-0/data/info.pb
14:10:22.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.207.190
14:10:22.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37257
14:10:22.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:10:22.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:10:22.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:10:22.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:10:22.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:10:22.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-0/logs
14:10:22.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:10:22.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:10:22.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:10:22.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:10:22.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:10:22.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 06466b25e294b4d7d455e6c56263318dc34a80ef
14:10:22.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:10:22.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 30 Aug 2025 13:43:24 UTC on 5fd53c4cbb9d
14:10:22.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7817
14:10:22.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:10:22.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:22.319638 21968 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:10:22.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:22.320966 21968 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:10:22.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:22.337391 21975 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:10:23.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:22.338269 21974 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:10:23.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:22.341828 21977 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:10:23.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:23.653653 21976 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1313 milliseconds
14:10:23.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:23.653779 21968 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:10:23.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:23.654954 21968 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:10:23.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:23.658033 21968 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:10:23.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:23.659466 21968 hybrid_clock.cc:648] HybridClock initialized: now 1756563023659422 us; error 60 us; skew 500 ppm
14:10:23.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:23.660465 21968 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:10:23.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 30 14:10:23 dist-test-slave-xkqm krb5kdc[21321](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756563023, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.207.190@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:10:23.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:23.666509 21968 init.cc:377] Logged in from keytab as kudu/127.20.207.190@KRBTEST.COM (short username kudu)
14:10:23.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:23.671854 21968 webserver.cc:480] Webserver started at http://127.20.207.190:37257/ using document root <none> and password file <none>
14:10:23.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:23.672797 21968 fs_manager.cc:362] Metadata directory not provided
14:10:23.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:23.672994 21968 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:10:23.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:23.681587 21968 fs_manager.cc:714] Time spent opening directory manager: real 0.006s user 0.004s sys 0.004s
14:10:23.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:23.686192 21989 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:10:23.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:23.687371 21968 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
14:10:23.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:23.687713 21968 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-0/data,/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-0/wal
14:10:23.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f34e5ae5629346b380aa3975ffecb1f3"
14:10:23.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-30 14:10:03 on dist-test-slave-xkqm"
14:10:23.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:23.689795 21968 fs_report.cc:389] FS layout report
14:10:23.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:10:23.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-0/wal
14:10:23.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-0/wal
14:10:23.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-0/data/data
14:10:23.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:10:23.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:10:23.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:10:23.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:10:23.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:10:23.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:10:23.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:10:23.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:10:23.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:10:23.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:10:23.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:10:23.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:23.744812 21968 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:10:23.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:23.750764 21968 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:10:23.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:23.751184 21968 kserver.cc:163] Server-wide thread pool size limit: 3276
14:10:23.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:23.825821 21968 rpc_server.cc:307] RPC server started. Bound to: 127.20.207.190:38271
14:10:23.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:23.826179 22040 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.20.207.190:38271 every 8 connection(s)
14:10:23.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:23.828681 21968 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-0/data/info.pb
14:10:23.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:23.830865 21310 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskBD8el9/build/tsan/bin/kudu as pid 21968
14:10:23.831 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:10:23.831 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.20.207.188:42709
14:10:23.832 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
14:10:23.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:23.834252 21310 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskBD8el9/build/tsan/bin/kudu
14:10:23.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskBD8el9/build/tsan/bin/kudu
14:10:23.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-2/wal
14:10:23.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-2/data
14:10:23.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:10:23.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:10:23.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:10:23.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:10:23.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:10:23.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:10:23.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-2/logs
14:10:23.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-2/data/info.pb
14:10:23.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:10:23.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:10:23.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:10:23.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:10:23.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:10:23.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:10:23.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:10:23.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:10:23.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:10:23.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:10:23.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:10:23.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:10:23.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.207.188:42709
14:10:23.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.207.188
14:10:23.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38937
14:10:23.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/krb5kdc/kudu.keytab
14:10:23.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.20.207.188
14:10:23.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:10:23.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:10:23.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:10:23.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
14:10:23.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.207.148:36627
14:10:23.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:10:23.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:10:23.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:10:23.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:10:23.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.207.190:38271,127.20.207.189:43023,127.20.207.188:42709
14:10:23.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:10:23.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34701/jwks.json
14:10:23.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
14:10:23.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:10:23.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:23.843197 22041 sys_catalog.cc:263] Verifying existing consensus state
14:10:23.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:23.848856 22041 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f34e5ae5629346b380aa3975ffecb1f3: Bootstrap starting.
14:10:23.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:23.887147 22041 log.cc:826] T 00000000000000000000000000000000 P f34e5ae5629346b380aa3975ffecb1f3: Log is configured to *not* fsync() on all Append() calls
14:10:23.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:23.904821 22041 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f34e5ae5629346b380aa3975ffecb1f3: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
14:10:23.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:23.905719 22041 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f34e5ae5629346b380aa3975ffecb1f3: Bootstrap complete.
14:10:23.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:23.929554 22041 raft_consensus.cc:357] T 00000000000000000000000000000000 P f34e5ae5629346b380aa3975ffecb1f3 [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f34e5ae5629346b380aa3975ffecb1f3" member_type: VOTER last_known_addr { host: "127.20.207.190" port: 38271 } } peers { permanent_uuid: "1fd5034219e348e289cf311cecde23f1" member_type: VOTER last_known_addr { host: "127.20.207.189" port: 43023 } } peers { permanent_uuid: "9c6aef56265e4e5ca5ee68e82eba93c9" member_type: VOTER last_known_addr { host: "127.20.207.188" port: 42709 } }
14:10:23.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:23.932168 22041 raft_consensus.cc:738] T 00000000000000000000000000000000 P f34e5ae5629346b380aa3975ffecb1f3 [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: f34e5ae5629346b380aa3975ffecb1f3, State: Initialized, Role: FOLLOWER
14:10:23.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:23.933101 22041 consensus_queue.cc:260] T 00000000000000000000000000000000 P f34e5ae5629346b380aa3975ffecb1f3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 2.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f34e5ae5629346b380aa3975ffecb1f3" member_type: VOTER last_known_addr { host: "127.20.207.190" port: 38271 } } peers { permanent_uuid: "1fd5034219e348e289cf311cecde23f1" member_type: VOTER last_known_addr { host: "127.20.207.189" port: 43023 } } peers { permanent_uuid: "9c6aef56265e4e5ca5ee68e82eba93c9" member_type: VOTER last_known_addr { host: "127.20.207.188" port: 42709 } }
14:10:23.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:23.936103 22045 sys_catalog.cc:455] T 00000000000000000000000000000000 P f34e5ae5629346b380aa3975ffecb1f3 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 2 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f34e5ae5629346b380aa3975ffecb1f3" member_type: VOTER last_known_addr { host: "127.20.207.190" port: 38271 } } peers { permanent_uuid: "1fd5034219e348e289cf311cecde23f1" member_type: VOTER last_known_addr { host: "127.20.207.189" port: 43023 } } peers { permanent_uuid: "9c6aef56265e4e5ca5ee68e82eba93c9" member_type: VOTER last_known_addr { host: "127.20.207.188" port: 42709 } } }
14:10:23.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:23.937107 22045 sys_catalog.cc:458] T 00000000000000000000000000000000 P f34e5ae5629346b380aa3975ffecb1f3 [sys.catalog]: This master's current role is: FOLLOWER
14:10:23.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:23.938046 22041 sys_catalog.cc:564] T 00000000000000000000000000000000 P f34e5ae5629346b380aa3975ffecb1f3 [sys.catalog]: configured and running, proceeding with master startup.
14:10:23.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:23.976867 22056 catalog_manager.cc:1261] Loaded cluster ID: d49b6b05d5d64b61a603c9b5cbaf4320
14:10:23.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:23.977181 22056 catalog_manager.cc:1554] T 00000000000000000000000000000000 P f34e5ae5629346b380aa3975ffecb1f3: loading cluster ID for follower catalog manager: success
14:10:23.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:23.986709 22056 catalog_manager.cc:1576] T 00000000000000000000000000000000 P f34e5ae5629346b380aa3975ffecb1f3: acquiring CA information for follower catalog manager: success
14:10:23.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 30 14:10:23 dist-test-slave-xkqm krb5kdc[21321](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756563023, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.207.190@KRBTEST.COM for kudu/127.20.207.190@KRBTEST.COM
14:10:23.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:23.995966 22056 catalog_manager.cc:1604] T 00000000000000000000000000000000 P f34e5ae5629346b380aa3975ffecb1f3: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
14:10:24.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:24.019620 21968 master_runner.cc:186] Error getting master registration for 127.20.207.189:43023: Network error: Client connection negotiation failed: client connection to 127.20.207.189:43023: connect: Connection refused (error 111),
14:10:24.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:24.048142 21968 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:10:24.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:24.198279 22043 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:10:24.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:24.199008 22043 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:10:24.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:24.199724 22043 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:10:24.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:24.233107 22043 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
14:10:24.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:24.233350 22043 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:10:24.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:24.233616 22043 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:10:24.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:24.233827 22043 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:10:24.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:24.234184 22043 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:10:24.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:24.234503 22043 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:10:24.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:24.234740 22043 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34701/jwks.json
14:10:24.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:24.280507 22043 master_runner.cc:386] Master server non-default flags:
14:10:24.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:10:24.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.207.148:36627
14:10:24.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:10:24.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:10:24.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-2/data
14:10:24.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-2/wal
14:10:24.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
14:10:24.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:10:24.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.207.190:38271,127.20.207.189:43023,127.20.207.188:42709
14:10:24.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/krb5kdc/kudu.keytab
14:10:24.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.20.207.188
14:10:24.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:10:24.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:10:24.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:10:24.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.207.188:42709
14:10:24.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:10:24.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:10:24.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:10:24.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34701/jwks.json
14:10:24.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:10:24.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:10:24.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:10:24.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:10:24.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:10:24.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-2/data/info.pb
14:10:24.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.207.188
14:10:24.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38937
14:10:24.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:10:24.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:10:24.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:10:24.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:10:24.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:10:24.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-2/logs
14:10:24.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:10:24.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:10:24.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:10:24.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:10:24.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:10:24.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 06466b25e294b4d7d455e6c56263318dc34a80ef
14:10:24.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:10:24.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 30 Aug 2025 13:43:24 UTC on 5fd53c4cbb9d
14:10:24.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7817
14:10:24.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:10:24.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:24.281805 22043 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:10:24.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:24.282974 22043 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:10:24.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:24.317094 22067 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:10:25.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:25.740720 22076 raft_consensus.cc:491] T 00000000000000000000000000000000 P f34e5ae5629346b380aa3975ffecb1f3 [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:10:25.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:25.741320 22076 raft_consensus.cc:513] T 00000000000000000000000000000000 P f34e5ae5629346b380aa3975ffecb1f3 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f34e5ae5629346b380aa3975ffecb1f3" member_type: VOTER last_known_addr { host: "127.20.207.190" port: 38271 } } peers { permanent_uuid: "1fd5034219e348e289cf311cecde23f1" member_type: VOTER last_known_addr { host: "127.20.207.189" port: 43023 } } peers { permanent_uuid: "9c6aef56265e4e5ca5ee68e82eba93c9" member_type: VOTER last_known_addr { host: "127.20.207.188" port: 42709 } }
14:10:25.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:25.753961 22076 leader_election.cc:290] T 00000000000000000000000000000000 P f34e5ae5629346b380aa3975ffecb1f3 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 1fd5034219e348e289cf311cecde23f1 (127.20.207.189:43023), 9c6aef56265e4e5ca5ee68e82eba93c9 (127.20.207.188:42709)
14:10:25.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:25.757436 21992 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.20.207.188:42709: connect: Connection refused (error 111)
14:10:25.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:25.773552 21992 leader_election.cc:336] T 00000000000000000000000000000000 P f34e5ae5629346b380aa3975ffecb1f3 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 9c6aef56265e4e5ca5ee68e82eba93c9 (127.20.207.188:42709): Network error: Client connection negotiation failed: client connection to 127.20.207.188:42709: connect: Connection refused (error 111)
14:10:25.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:25.788460 21993 leader_election.cc:336] T 00000000000000000000000000000000 P f34e5ae5629346b380aa3975ffecb1f3 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 1fd5034219e348e289cf311cecde23f1 (127.20.207.189:43023): Network error: Client connection negotiation failed: client connection to 127.20.207.189:43023: connect: Connection refused (error 111)
14:10:25.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:25.789084 21993 leader_election.cc:304] T 00000000000000000000000000000000 P f34e5ae5629346b380aa3975ffecb1f3 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: f34e5ae5629346b380aa3975ffecb1f3; no voters: 1fd5034219e348e289cf311cecde23f1, 9c6aef56265e4e5ca5ee68e82eba93c9
14:10:25.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:25.790145 22076 raft_consensus.cc:2747] T 00000000000000000000000000000000 P f34e5ae5629346b380aa3975ffecb1f3 [term 2 FOLLOWER]: Leader pre-election lost for term 3. Reason: could not achieve majority
14:10:26.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:25.713634 22065 debug-util.cc:398] Leaking SignalData structure 0x7b0800037200 after lost signal to thread 22043
14:10:26.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:26.072870 22065 kernel_stack_watchdog.cc:198] Thread 22043 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 382ms:
14:10:26.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
14:10:26.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
14:10:26.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
14:10:26.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
14:10:26.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:24.338264 22068 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:10:26.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:26.077540 22043 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.749s user 0.655s sys 1.048s
14:10:26.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:26.077847 22043 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.749s user 0.655s sys 1.048s
14:10:26.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:26.078112 22043 server_base.cc:1047] running on GCE node
14:10:26.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:26.079365 22070 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:10:26.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:26.080680 22043 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:10:26.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:26.083633 22043 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:10:26.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:26.085044 22043 hybrid_clock.cc:648] HybridClock initialized: now 1756563026084974 us; error 84 us; skew 500 ppm
14:10:26.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:26.086004 22043 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:10:26.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 30 14:10:26 dist-test-slave-xkqm krb5kdc[21321](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756563026, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.207.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:10:26.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:26.091874 22043 init.cc:377] Logged in from keytab as kudu/127.20.207.188@KRBTEST.COM (short username kudu)
14:10:26.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:26.098232 22043 webserver.cc:480] Webserver started at http://127.20.207.188:38937/ using document root <none> and password file <none>
14:10:26.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:26.099318 22043 fs_manager.cc:362] Metadata directory not provided
14:10:26.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:26.099640 22043 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:10:26.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:26.111229 22043 fs_manager.cc:714] Time spent opening directory manager: real 0.007s user 0.005s sys 0.005s
14:10:26.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:26.117444 22086 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:10:26.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:26.118852 22043 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.002s sys 0.002s
14:10:26.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:26.119396 22043 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-2/data,/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-2/wal
14:10:26.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9c6aef56265e4e5ca5ee68e82eba93c9"
14:10:26.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-30 14:10:07 on dist-test-slave-xkqm"
14:10:26.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:26.122350 22043 fs_report.cc:389] FS layout report
14:10:26.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:10:26.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-2/wal
14:10:26.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-2/wal
14:10:26.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-2/data/data
14:10:26.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:10:26.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:10:26.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:10:26.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:10:26.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:10:26.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:10:26.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:10:26.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:10:26.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:10:26.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:10:26.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:10:26.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:26.229353 22043 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:10:26.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:26.237290 22043 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:10:26.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:26.237864 22043 kserver.cc:163] Server-wide thread pool size limit: 3276
14:10:26.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:26.353940 22043 rpc_server.cc:307] RPC server started. Bound to: 127.20.207.188:42709
14:10:26.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:26.354079 22137 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.20.207.188:42709 every 8 connection(s)
14:10:26.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:26.356873 22043 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-2/data/info.pb
14:10:26.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:26.363543 21310 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskBD8el9/build/tsan/bin/kudu as pid 22043
14:10:26.364 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:10:26.364 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.20.207.189:43023
14:10:26.364 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
14:10:26.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:26.365835 21310 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskBD8el9/build/tsan/bin/kudu
14:10:26.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskBD8el9/build/tsan/bin/kudu
14:10:26.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-1/wal
14:10:26.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-1/data
14:10:26.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:10:26.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:10:26.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:10:26.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:10:26.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:10:26.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:10:26.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-1/logs
14:10:26.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-1/data/info.pb
14:10:26.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:10:26.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:10:26.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:10:26.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:10:26.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:10:26.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:10:26.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:10:26.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:10:26.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:10:26.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:10:26.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:10:26.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:10:26.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.207.189:43023
14:10:26.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.207.189
14:10:26.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=43607
14:10:26.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/krb5kdc/kudu.keytab
14:10:26.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.20.207.189
14:10:26.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:10:26.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:10:26.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:10:26.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
14:10:26.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.207.148:36627
14:10:26.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:10:26.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:10:26.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:10:26.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:10:26.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.207.190:38271,127.20.207.189:43023,127.20.207.188:42709
14:10:26.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:10:26.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34701/jwks.json
14:10:26.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
14:10:26.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:10:26.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:26.371508 22138 sys_catalog.cc:263] Verifying existing consensus state
14:10:26.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:26.380504 22138 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9c6aef56265e4e5ca5ee68e82eba93c9: Bootstrap starting.
14:10:26.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:26.437299 22138 log.cc:826] T 00000000000000000000000000000000 P 9c6aef56265e4e5ca5ee68e82eba93c9: Log is configured to *not* fsync() on all Append() calls
14:10:26.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:26.461557 22138 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9c6aef56265e4e5ca5ee68e82eba93c9: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
14:10:26.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:26.462570 22138 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9c6aef56265e4e5ca5ee68e82eba93c9: Bootstrap complete.
14:10:26.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:26.498996 22138 raft_consensus.cc:357] T 00000000000000000000000000000000 P 9c6aef56265e4e5ca5ee68e82eba93c9 [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f34e5ae5629346b380aa3975ffecb1f3" member_type: VOTER last_known_addr { host: "127.20.207.190" port: 38271 } } peers { permanent_uuid: "1fd5034219e348e289cf311cecde23f1" member_type: VOTER last_known_addr { host: "127.20.207.189" port: 43023 } } peers { permanent_uuid: "9c6aef56265e4e5ca5ee68e82eba93c9" member_type: VOTER last_known_addr { host: "127.20.207.188" port: 42709 } }
14:10:26.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:26.503453 22138 raft_consensus.cc:738] T 00000000000000000000000000000000 P 9c6aef56265e4e5ca5ee68e82eba93c9 [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9c6aef56265e4e5ca5ee68e82eba93c9, State: Initialized, Role: FOLLOWER
14:10:26.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:26.505028 22138 consensus_queue.cc:260] T 00000000000000000000000000000000 P 9c6aef56265e4e5ca5ee68e82eba93c9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 2.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f34e5ae5629346b380aa3975ffecb1f3" member_type: VOTER last_known_addr { host: "127.20.207.190" port: 38271 } } peers { permanent_uuid: "1fd5034219e348e289cf311cecde23f1" member_type: VOTER last_known_addr { host: "127.20.207.189" port: 43023 } } peers { permanent_uuid: "9c6aef56265e4e5ca5ee68e82eba93c9" member_type: VOTER last_known_addr { host: "127.20.207.188" port: 42709 } }
14:10:26.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:26.514583 22143 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9c6aef56265e4e5ca5ee68e82eba93c9 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 2 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f34e5ae5629346b380aa3975ffecb1f3" member_type: VOTER last_known_addr { host: "127.20.207.190" port: 38271 } } peers { permanent_uuid: "1fd5034219e348e289cf311cecde23f1" member_type: VOTER last_known_addr { host: "127.20.207.189" port: 43023 } } peers { permanent_uuid: "9c6aef56265e4e5ca5ee68e82eba93c9" member_type: VOTER last_known_addr { host: "127.20.207.188" port: 42709 } } }
14:10:26.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:26.516045 22143 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9c6aef56265e4e5ca5ee68e82eba93c9 [sys.catalog]: This master's current role is: FOLLOWER
14:10:26.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:26.516587 22138 sys_catalog.cc:564] T 00000000000000000000000000000000 P 9c6aef56265e4e5ca5ee68e82eba93c9 [sys.catalog]: configured and running, proceeding with master startup.
14:10:26.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:26.590175 22154 catalog_manager.cc:1261] Loaded cluster ID: d49b6b05d5d64b61a603c9b5cbaf4320
14:10:26.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:26.590538 22154 catalog_manager.cc:1554] T 00000000000000000000000000000000 P 9c6aef56265e4e5ca5ee68e82eba93c9: loading cluster ID for follower catalog manager: success
14:10:26.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:26.605475 22154 catalog_manager.cc:1576] T 00000000000000000000000000000000 P 9c6aef56265e4e5ca5ee68e82eba93c9: acquiring CA information for follower catalog manager: success
14:10:26.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:26.610966 22154 catalog_manager.cc:1604] T 00000000000000000000000000000000 P 9c6aef56265e4e5ca5ee68e82eba93c9: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
14:10:26.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 30 14:10:26 dist-test-slave-xkqm krb5kdc[21321](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756563026, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.207.188@KRBTEST.COM for kudu/127.20.207.190@KRBTEST.COM
14:10:26.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:26.643568 22043 master_runner.cc:186] Error getting master registration for 127.20.207.189:43023: Network error: Client connection negotiation failed: client connection to 127.20.207.189:43023: connect: Connection refused (error 111),
14:10:26.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:26.673370 22043 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:10:26.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:26.779999 22140 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:10:26.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:26.780669 22140 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:10:26.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:26.781355 22140 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:10:26.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:26.817284 22140 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
14:10:26.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:26.817662 22140 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:10:26.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:26.818109 22140 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:10:26.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:26.818397 22140 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:10:26.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:26.818670 22140 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:10:26.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:26.818960 22140 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:10:26.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:26.819166 22140 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34701/jwks.json
14:10:26.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:26.873638 22140 master_runner.cc:386] Master server non-default flags:
14:10:26.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:10:26.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.207.148:36627
14:10:26.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:10:26.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:10:26.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-1/data
14:10:26.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-1/wal
14:10:26.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
14:10:26.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:10:26.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.207.190:38271,127.20.207.189:43023,127.20.207.188:42709
14:10:26.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/krb5kdc/kudu.keytab
14:10:26.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.20.207.189
14:10:26.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:10:26.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:10:26.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:10:26.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.207.189:43023
14:10:26.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:10:26.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:10:26.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:10:26.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34701/jwks.json
14:10:26.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:10:26.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:10:26.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:10:26.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:10:26.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:10:26.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-1/data/info.pb
14:10:26.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.207.189
14:10:26.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=43607
14:10:26.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:10:26.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:10:26.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:10:26.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:10:26.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:10:26.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-1/logs
14:10:26.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:10:26.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:10:26.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:10:26.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:10:26.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:10:26.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 06466b25e294b4d7d455e6c56263318dc34a80ef
14:10:26.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:10:26.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 30 Aug 2025 13:43:24 UTC on 5fd53c4cbb9d
14:10:26.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7817
14:10:26.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:10:26.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:26.875262 22140 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:10:26.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:26.876520 22140 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:10:26.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:26.888653 22164 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:10:27.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:27.214816 21154 debug-util.cc:398] Leaking SignalData structure 0x7b08000ac2a0 after lost signal to thread 21026
14:10:27.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:27.216516 21154 debug-util.cc:398] Leaking SignalData structure 0x7b08000baee0 after lost signal to thread 21157
14:10:27.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:27.705538 22170 raft_consensus.cc:491] T 00000000000000000000000000000000 P 9c6aef56265e4e5ca5ee68e82eba93c9 [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:10:27.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:27.706173 22170 raft_consensus.cc:513] T 00000000000000000000000000000000 P 9c6aef56265e4e5ca5ee68e82eba93c9 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f34e5ae5629346b380aa3975ffecb1f3" member_type: VOTER last_known_addr { host: "127.20.207.190" port: 38271 } } peers { permanent_uuid: "1fd5034219e348e289cf311cecde23f1" member_type: VOTER last_known_addr { host: "127.20.207.189" port: 43023 } } peers { permanent_uuid: "9c6aef56265e4e5ca5ee68e82eba93c9" member_type: VOTER last_known_addr { host: "127.20.207.188" port: 42709 } }
14:10:27.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:27.710059 22016 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "9c6aef56265e4e5ca5ee68e82eba93c9" candidate_term: 3 candidate_status { last_received { term: 2 index: 5 } } ignore_live_leader: false dest_uuid: "f34e5ae5629346b380aa3975ffecb1f3" is_pre_election: true
14:10:27.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:27.710855 22016 raft_consensus.cc:2466] T 00000000000000000000000000000000 P f34e5ae5629346b380aa3975ffecb1f3 [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 9c6aef56265e4e5ca5ee68e82eba93c9 in term 2.
14:10:27.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:27.712303 22090 leader_election.cc:304] T 00000000000000000000000000000000 P 9c6aef56265e4e5ca5ee68e82eba93c9 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 9c6aef56265e4e5ca5ee68e82eba93c9, f34e5ae5629346b380aa3975ffecb1f3; no voters:
14:10:27.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:27.719722 22171 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 9c6aef56265e4e5ca5ee68e82eba93c9 [term 2 FOLLOWER]: Leader pre-election won for term 3
14:10:27.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:27.720310 22171 raft_consensus.cc:491] T 00000000000000000000000000000000 P 9c6aef56265e4e5ca5ee68e82eba93c9 [term 2 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
14:10:27.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:27.723317 22171 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 9c6aef56265e4e5ca5ee68e82eba93c9 [term 2 FOLLOWER]: Advancing to term 3
14:10:27.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:27.733284 22170 leader_election.cc:290] T 00000000000000000000000000000000 P 9c6aef56265e4e5ca5ee68e82eba93c9 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers f34e5ae5629346b380aa3975ffecb1f3 (127.20.207.190:38271), 1fd5034219e348e289cf311cecde23f1 (127.20.207.189:43023)
14:10:27.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:27.738101 22090 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.20.207.189:43023: connect: Connection refused (error 111)
14:10:27.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:27.751981 22090 leader_election.cc:336] T 00000000000000000000000000000000 P 9c6aef56265e4e5ca5ee68e82eba93c9 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 1fd5034219e348e289cf311cecde23f1 (127.20.207.189:43023): Network error: Client connection negotiation failed: client connection to 127.20.207.189:43023: connect: Connection refused (error 111)
14:10:27.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:27.755158 22171 raft_consensus.cc:513] T 00000000000000000000000000000000 P 9c6aef56265e4e5ca5ee68e82eba93c9 [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f34e5ae5629346b380aa3975ffecb1f3" member_type: VOTER last_known_addr { host: "127.20.207.190" port: 38271 } } peers { permanent_uuid: "1fd5034219e348e289cf311cecde23f1" member_type: VOTER last_known_addr { host: "127.20.207.189" port: 43023 } } peers { permanent_uuid: "9c6aef56265e4e5ca5ee68e82eba93c9" member_type: VOTER last_known_addr { host: "127.20.207.188" port: 42709 } }
14:10:27.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:27.762094 22171 leader_election.cc:290] T 00000000000000000000000000000000 P 9c6aef56265e4e5ca5ee68e82eba93c9 [CANDIDATE]: Term 3 election: Requested vote from peers f34e5ae5629346b380aa3975ffecb1f3 (127.20.207.190:38271), 1fd5034219e348e289cf311cecde23f1 (127.20.207.189:43023)
14:10:27.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:27.764695 22016 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "9c6aef56265e4e5ca5ee68e82eba93c9" candidate_term: 3 candidate_status { last_received { term: 2 index: 5 } } ignore_live_leader: false dest_uuid: "f34e5ae5629346b380aa3975ffecb1f3"
14:10:27.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:27.765272 22016 raft_consensus.cc:3058] T 00000000000000000000000000000000 P f34e5ae5629346b380aa3975ffecb1f3 [term 2 FOLLOWER]: Advancing to term 3
14:10:27.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:27.776737 22090 leader_election.cc:336] T 00000000000000000000000000000000 P 9c6aef56265e4e5ca5ee68e82eba93c9 [CANDIDATE]: Term 3 election: RPC error from VoteRequest() call to peer 1fd5034219e348e289cf311cecde23f1 (127.20.207.189:43023): Network error: Client connection negotiation failed: client connection to 127.20.207.189:43023: connect: Connection refused (error 111)
14:10:27.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:27.782415 22016 raft_consensus.cc:2466] T 00000000000000000000000000000000 P f34e5ae5629346b380aa3975ffecb1f3 [term 3 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 9c6aef56265e4e5ca5ee68e82eba93c9 in term 3.
14:10:27.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:27.783627 22090 leader_election.cc:304] T 00000000000000000000000000000000 P 9c6aef56265e4e5ca5ee68e82eba93c9 [CANDIDATE]: Term 3 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 9c6aef56265e4e5ca5ee68e82eba93c9, f34e5ae5629346b380aa3975ffecb1f3; no voters: 1fd5034219e348e289cf311cecde23f1
14:10:27.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:27.784561 22171 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 9c6aef56265e4e5ca5ee68e82eba93c9 [term 3 FOLLOWER]: Leader election won for term 3
14:10:27.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:27.785130 22171 raft_consensus.cc:695] T 00000000000000000000000000000000 P 9c6aef56265e4e5ca5ee68e82eba93c9 [term 3 LEADER]: Becoming Leader. State: Replica: 9c6aef56265e4e5ca5ee68e82eba93c9, State: Running, Role: LEADER
14:10:27.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:27.786302 22171 consensus_queue.cc:237] T 00000000000000000000000000000000 P 9c6aef56265e4e5ca5ee68e82eba93c9 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 2.5, Last appended by leader: 5, Current term: 3, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f34e5ae5629346b380aa3975ffecb1f3" member_type: VOTER last_known_addr { host: "127.20.207.190" port: 38271 } } peers { permanent_uuid: "1fd5034219e348e289cf311cecde23f1" member_type: VOTER last_known_addr { host: "127.20.207.189" port: 43023 } } peers { permanent_uuid: "9c6aef56265e4e5ca5ee68e82eba93c9" member_type: VOTER last_known_addr { host: "127.20.207.188" port: 42709 } }
14:10:27.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:27.813071 22170 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9c6aef56265e4e5ca5ee68e82eba93c9 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 9c6aef56265e4e5ca5ee68e82eba93c9. Latest consensus state: current_term: 3 leader_uuid: "9c6aef56265e4e5ca5ee68e82eba93c9" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f34e5ae5629346b380aa3975ffecb1f3" member_type: VOTER last_known_addr { host: "127.20.207.190" port: 38271 } } peers { permanent_uuid: "1fd5034219e348e289cf311cecde23f1" member_type: VOTER last_known_addr { host: "127.20.207.189" port: 43023 } } peers { permanent_uuid: "9c6aef56265e4e5ca5ee68e82eba93c9" member_type: VOTER last_known_addr { host: "127.20.207.188" port: 42709 } } }
14:10:27.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:27.813843 22170 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9c6aef56265e4e5ca5ee68e82eba93c9 [sys.catalog]: This master's current role is: LEADER
14:10:27.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:27.834401 22175 catalog_manager.cc:1477] Loading table and tablet metadata into memory...
14:10:27.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:27.877521 22175 catalog_manager.cc:1486] Initializing Kudu cluster ID...
14:10:27.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:27.900251 22175 catalog_manager.cc:1261] Loaded cluster ID: d49b6b05d5d64b61a603c9b5cbaf4320
14:10:27.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:27.900657 22175 catalog_manager.cc:1497] Initializing Kudu internal certificate authority...
14:10:27.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:27.925823 22175 catalog_manager.cc:1506] Loading token signing keys...
14:10:27.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:27.948606 22175 catalog_manager.cc:5966] T 00000000000000000000000000000000 P 9c6aef56265e4e5ca5ee68e82eba93c9: Loaded TSK: 0
14:10:27.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:27.955699 22175 catalog_manager.cc:1516] Initializing in-progress tserver states...
14:10:28.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:28.219135 22016 raft_consensus.cc:1273] T 00000000000000000000000000000000 P f34e5ae5629346b380aa3975ffecb1f3 [term 3 FOLLOWER]: Refusing update from remote peer 9c6aef56265e4e5ca5ee68e82eba93c9: Log matching property violated. Preceding OpId in replica: term: 2 index: 5. Preceding OpId from leader: term: 3 index: 6. (index mismatch)
14:10:28.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:28.220919 22170 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 9c6aef56265e4e5ca5ee68e82eba93c9 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f34e5ae5629346b380aa3975ffecb1f3" member_type: VOTER last_known_addr { host: "127.20.207.190" port: 38271 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
14:10:28.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:28.258417 22176 sys_catalog.cc:455] T 00000000000000000000000000000000 P f34e5ae5629346b380aa3975ffecb1f3 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 9c6aef56265e4e5ca5ee68e82eba93c9. Latest consensus state: current_term: 3 leader_uuid: "9c6aef56265e4e5ca5ee68e82eba93c9" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f34e5ae5629346b380aa3975ffecb1f3" member_type: VOTER last_known_addr { host: "127.20.207.190" port: 38271 } } peers { permanent_uuid: "1fd5034219e348e289cf311cecde23f1" member_type: VOTER last_known_addr { host: "127.20.207.189" port: 43023 } } peers { permanent_uuid: "9c6aef56265e4e5ca5ee68e82eba93c9" member_type: VOTER last_known_addr { host: "127.20.207.188" port: 42709 } } }
14:10:28.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:28.259403 22176 sys_catalog.cc:458] T 00000000000000000000000000000000 P f34e5ae5629346b380aa3975ffecb1f3 [sys.catalog]: This master's current role is: FOLLOWER
14:10:28.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:28.279248 22176 sys_catalog.cc:455] T 00000000000000000000000000000000 P f34e5ae5629346b380aa3975ffecb1f3 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "9c6aef56265e4e5ca5ee68e82eba93c9" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f34e5ae5629346b380aa3975ffecb1f3" member_type: VOTER last_known_addr { host: "127.20.207.190" port: 38271 } } peers { permanent_uuid: "1fd5034219e348e289cf311cecde23f1" member_type: VOTER last_known_addr { host: "127.20.207.189" port: 43023 } } peers { permanent_uuid: "9c6aef56265e4e5ca5ee68e82eba93c9" member_type: VOTER last_known_addr { host: "127.20.207.188" port: 42709 } } }
14:10:28.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:28.280119 22176 sys_catalog.cc:458] T 00000000000000000000000000000000 P f34e5ae5629346b380aa3975ffecb1f3 [sys.catalog]: This master's current role is: FOLLOWER
14:10:28.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:28.306041 22170 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9c6aef56265e4e5ca5ee68e82eba93c9 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 3 leader_uuid: "9c6aef56265e4e5ca5ee68e82eba93c9" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f34e5ae5629346b380aa3975ffecb1f3" member_type: VOTER last_known_addr { host: "127.20.207.190" port: 38271 } } peers { permanent_uuid: "1fd5034219e348e289cf311cecde23f1" member_type: VOTER last_known_addr { host: "127.20.207.189" port: 43023 } } peers { permanent_uuid: "9c6aef56265e4e5ca5ee68e82eba93c9" member_type: VOTER last_known_addr { host: "127.20.207.188" port: 42709 } } }
14:10:28.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:28.307061 22170 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9c6aef56265e4e5ca5ee68e82eba93c9 [sys.catalog]: This master's current role is: LEADER
14:10:28.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:28.321501 22090 consensus_peers.cc:489] T 00000000000000000000000000000000 P 9c6aef56265e4e5ca5ee68e82eba93c9 -> Peer 1fd5034219e348e289cf311cecde23f1 (127.20.207.189:43023): Couldn't send request to peer 1fd5034219e348e289cf311cecde23f1. Status: Network error: Client connection negotiation failed: client connection to 127.20.207.189:43023: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
14:10:28.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:26.889814 22165 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:10:28.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:28.290499 22163 debug-util.cc:398] Leaking SignalData structure 0x7b0800037200 after lost signal to thread 22140
14:10:28.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:28.915753 22140 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 2.026s user 0.621s sys 0.942s
14:10:28.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:28.917574 22140 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 2.028s user 0.621s sys 0.943s
14:10:28.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:28.918963 22167 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:10:28.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:28.922503 22166 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2026 milliseconds
14:10:28.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:28.922508 22140 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:10:28.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:28.924269 22140 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:10:28.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:28.927479 22140 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:10:28.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:28.929582 22140 hybrid_clock.cc:648] HybridClock initialized: now 1756563028929538 us; error 42 us; skew 500 ppm
14:10:28.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:28.930615 22140 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:10:28.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 30 14:10:28 dist-test-slave-xkqm krb5kdc[21321](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756563028, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.207.189@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:10:28.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:28.938181 22140 init.cc:377] Logged in from keytab as kudu/127.20.207.189@KRBTEST.COM (short username kudu)
14:10:28.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:28.944133 22140 webserver.cc:480] Webserver started at http://127.20.207.189:43607/ using document root <none> and password file <none>
14:10:28.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:28.945405 22140 fs_manager.cc:362] Metadata directory not provided
14:10:28.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:28.945767 22140 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:10:28.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:28.955785 22140 fs_manager.cc:714] Time spent opening directory manager: real 0.006s user 0.004s sys 0.000s
14:10:28.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:28.960484 22193 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:10:28.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:28.961740 22140 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
14:10:28.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:28.962085 22140 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-1/data,/tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-1/wal
14:10:28.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1fd5034219e348e289cf311cecde23f1"
14:10:28.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-30 14:10:05 on dist-test-slave-xkqm"
14:10:28.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:28.964036 22140 fs_report.cc:389] FS layout report
14:10:28.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:10:28.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-1/wal
14:10:28.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-1/wal
14:10:28.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-1/data/data
14:10:28.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:10:28.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:10:28.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:10:28.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:10:28.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:10:28.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:10:28.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:10:28.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:10:28.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:10:28.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:10:28.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:10:29.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:29.025559 22140 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:10:29.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:29.031774 22140 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:10:29.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:29.032325 22140 kserver.cc:163] Server-wide thread pool size limit: 3276
14:10:29.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:29.056717 21968 master_runner.cc:186] Error getting master registration for 127.20.207.189:43023: Network error: Client connection negotiation failed: client connection to 127.20.207.189:43023: connect: Connection refused (error 111),
14:10:29.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:29.057286 21968 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:10:29.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:29.135321 22140 rpc_server.cc:307] RPC server started. Bound to: 127.20.207.189:43023
14:10:29.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:29.135465 22245 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.20.207.189:43023 every 8 connection(s)
14:10:29.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:29.138984 22140 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/master-1/data/info.pb
14:10:29.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:29.146282 21310 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskBD8el9/build/tsan/bin/kudu as pid 22140
14:10:29.147 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:10:29.148 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.20.207.190:38271(127.20.207.190:38271)] connecting to peer
14:10:29.150 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.20.207.190:38271(127.20.207.190:38271)] Successfully connected to peer
14:10:29.153 [DEBUG - kudu-nio-1] (SecurityContext.java:222) Refreshing Kerberos credentials...
14:10:29.154 [DEBUG - kudu-nio-1] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskBD8el9/test-tmp/mini-kudu-cluster4618415527907221443/krb5kdc/krb5cc
14:10:29.155 [DEBUG - kudu-nio-1] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
14:10:29.155 [WARN - kudu-nio-1] (SecurityContext.java:231) Tried to refresh Kerberos credentials but was unable to re-login from ticket cache
14:10:29.156 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.20.207.190:38271(127.20.207.190:38271)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
14:10:29.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:29.156316 22246 sys_catalog.cc:263] Verifying existing consensus state
14:10:29.157 [DEBUG - main] (Connection.java:575) [peer master-127.20.207.188:42709(127.20.207.188:42709)] disconnecting while in state TERMINATED
14:10:29.157 [DEBUG - main] (Connection.java:575) [peer master-127.20.207.190:38271(127.20.207.190:38271)] disconnecting while in state TERMINATED
14:10:29.157 [DEBUG - main] (Connection.java:575) [peer master-127.20.207.189:43023(127.20.207.189:43023)] disconnecting while in state TERMINATED
14:10:29.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250830 14:10:29.160054 22248 negotiation.cc:337] Failed RPC negotiation. Trace:
14:10:29.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0830 14:10:29.150124 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:44256 (local address 127.20.207.190:38271)
14:10:29.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0830 14:10:29.152533 (+ 2409us) server_negotiation.cc:206] Beginning negotiation
14:10:29.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0830 14:10:29.152551 (+ 18us) server_negotiation.cc:399] Waiting for connection header
14:10:29.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0830 14:10:29.152597 (+ 46us) server_negotiation.cc:407] Connection header received
14:10:29.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0830 14:10:29.152735 (+ 138us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
14:10:29.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0830 14:10:29.152753 (+ 18us) server_negotiation.cc:461] Received NEGOTIATE request from client
14:10:29.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0830 14:10:29.152922 (+ 169us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
14:10:29.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0830 14:10:29.153083 (+ 161us) server_negotiation.cc:226] Negotiated authn=SASL
14:10:29.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0830 14:10:29.158643 (+ 5560us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:44256: BlockingRecv error: recv got EOF from 127.0.0.1:44256 (error 108)
14:10:29.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":1929,"thread_start_us":1679,"threads_started":1}
14:10:29.160 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2704) Releasing all remaining resources
14:10:29.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:29.164623 21310 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskBD8el9/build/tsan/bin/kudu with pid 21968
14:10:29.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:29.166046 22246 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1fd5034219e348e289cf311cecde23f1: Bootstrap starting.
14:10:29.168 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.20.207.190:38271(127.20.207.190:38271)] handling channelInactive
14:10:29.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:29.199402 21310 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskBD8el9/build/tsan/bin/kudu with pid 22140
14:10:29.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:29.225775 21310 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskBD8el9/build/tsan/bin/kudu with pid 22043
14:10:30.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-08-30T14:10:30Z chronyd exiting
14:10:31.469 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
14:10:31.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:31.470942 20583 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskBD8el9/build/tsan/bin/kudu with pid 20863
14:10:31.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:31.498711 20583 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskBD8el9/build/tsan/bin/kudu with pid 21025
14:10:31.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:31.527277 20583 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskBD8el9/build/tsan/bin/kudu with pid 21169
14:10:31.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:31.554860 20583 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskBD8el9/build/tsan/bin/kudu with pid 20595
14:10:31.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:31.585093 20583 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskBD8el9/build/tsan/bin/kudu with pid 20665
14:10:31.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250830 14:10:31.616994 20583 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskBD8el9/build/tsan/bin/kudu with pid 20740
14:10:32.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-08-30T14:10:32Z chronyd exiting
14:10:33.804 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testRenewAndReacquireKeberosCredentials: failed attempt 1
org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testRenewAndReacquireKeberosCredentials(TestSecurity.java:346) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.Negotiator.chooseAndInitializeSaslMech(Negotiator.java:565) ~[main/:?]
at org.apache.kudu.client.Negotiator.handleNegotiateResponse(Negotiator.java:433) ~[main/:?]
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:367) ~[main/:?]
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:94) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testRenewAndReacquireKeberosCredentials(TestSecurity.java:346)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:748)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.Negotiator.chooseAndInitializeSaslMech(Negotiator.java:565)
at org.apache.kudu.client.Negotiator.handleNegotiateResponse(Negotiator.java:433)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:367)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:94)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
14:10:34.034 [ERROR - main] (RetryRule.java:253) org.apache.kudu.client.TestSecurity.testRenewAndReacquireKeberosCredentials: giving up after 1 attempts
14:10:34.035 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testRenewAndReacquireKeberosCredentials: failed attempt 1
org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testRenewAndReacquireKeberosCredentials(TestSecurity.java:346) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.Negotiator.chooseAndInitializeSaslMech(Negotiator.java:565) ~[main/:?]
at org.apache.kudu.client.Negotiator.handleNegotiateResponse(Negotiator.java:433) ~[main/:?]
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:367) ~[main/:?]
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:94) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testRenewAndReacquireKeberosCredentials(TestSecurity.java:346)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:748)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.Negotiator.chooseAndInitializeSaslMech(Negotiator.java:565)
at org.apache.kudu.client.Negotiator.handleNegotiateResponse(Negotiator.java:433)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:367)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:94)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more