08:03:42.204 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
08:03:42.215 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskf7pa4T/test-tmp
08:03:42.221 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
08:03:42.224 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskf7pa4T/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
08:03:42.434 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster11388303748095761808"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
08:03:42.623 [DEBUG - main] (MiniKuduCluster.java:195) Response:
08:03:42.624 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
08:03:42.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:42.628398 23696 mini_oidc.cc:150] Starting JWKS server
08:03:42.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:42.632056 23696 webserver.cc:492] Webserver started at http://127.0.0.1:43625/ using document root <none> and password file <none>
08:03:42.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:42.632603 23696 mini_oidc.cc:186] Starting OIDC Discovery server
08:03:42.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:42.634698 23696 webserver.cc:492] Webserver started at http://127.0.0.1:33989/ using document root <none> and password file <none>
08:03:42.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-04-30T08:03:42Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
08:03:42.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-04-30T08:03:42Z Disabled control of system clock
08:03:42.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:42.696534 23696 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskf7pa4T/build/tsan/bin/kudu
08:03:42.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskf7pa4T/build/tsan/bin/kudu
08:03:42.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster11388303748095761808/master-0/wal
08:03:42.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster11388303748095761808/master-0/data
08:03:42.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:03:42.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:03:42.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:03:42.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:03:42.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:03:42.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:03:42.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster11388303748095761808/master-0/logs
08:03:42.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster11388303748095761808/master-0/data/info.pb
08:03:42.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:03:42.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:03:42.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:03:42.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:03:42.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:03:42.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:03:42.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:03:42.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:03:42.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:03:42.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:03:42.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:03:42.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:03:42.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.36.62:46005
08:03:42.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.36.62
08:03:42.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:03:42.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.36.20:36785
08:03:42.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:03:42.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:03:42.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:03:42.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:03:42.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.36.62:46005,127.23.36.61:41985,127.23.36.60:37459
08:03:42.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:03:42.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43625/jwks.json with env {}
08:03:43.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:03:43.001705 23708 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
08:03:43.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:03:43.002367 23708 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:03:43.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:03:43.002907 23708 flags.cc:432] Enabled unsafe flag: --never_fsync=true
08:03:43.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:03:43.036592 23708 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
08:03:43.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:03:43.036909 23708 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
08:03:43.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:03:43.037221 23708 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
08:03:43.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:03:43.037484 23708 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
08:03:43.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:03:43.037839 23708 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
08:03:43.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:03:43.038086 23708 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:43625/jwks.json
08:03:43.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:43.075826 23708 master_runner.cc:386] Master server non-default flags:
08:03:43.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:03:43.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.36.20:36785
08:03:43.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:03:43.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:03:43.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster11388303748095761808/master-0/data
08:03:43.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster11388303748095761808/master-0/wal
08:03:43.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:03:43.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.36.62:46005,127.23.36.61:41985,127.23.36.60:37459
08:03:43.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:03:43.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:03:43.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:03:43.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.36.62:46005
08:03:43.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:03:43.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:03:43.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:03:43.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43625/jwks.json
08:03:43.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:03:43.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:03:43.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster11388303748095761808/master-0/data/info.pb
08:03:43.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.36.62
08:03:43.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:03:43.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:03:43.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:03:43.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:03:43.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:03:43.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:03:43.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster11388303748095761808/master-0/logs
08:03:43.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:03:43.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:03:43.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:03:43.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:03:43.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:03:43.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 8537ed5bb5e0e6ac1e481ff961e6b9d08cce7671
08:03:43.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:03:43.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 30 Apr 2026 07:43:22 UTC on e7f111948823
08:03:43.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11675
08:03:43.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:03:43.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:43.077235 23708 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:03:43.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:43.079442 23708 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:03:43.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:03:43.092851 23714 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:44.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:03:43.096099 23715 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:44.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:03:43.096707 23717 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:44.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:03:44.287001 23716 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
08:03:44.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:44.287163 23708 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
08:03:44.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:44.291585 23708 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:03:44.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:44.294919 23708 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:03:44.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:44.296391 23708 hybrid_clock.cc:648] HybridClock initialized: now 1777536224296337 us; error 63 us; skew 500 ppm
08:03:44.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:44.297286 23708 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:03:44.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:44.304997 23708 webserver.cc:492] Webserver started at http://127.23.36.62:41239/ using document root <none> and password file <none>
08:03:44.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:44.306480 23708 fs_manager.cc:362] Metadata directory not provided
08:03:44.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:44.306792 23708 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:03:44.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:44.307314 23708 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
08:03:44.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:44.311700 23708 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster11388303748095761808/master-0/data/instance:
08:03:44.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "91cc0cbb90c5488980c6a0da24388997"
08:03:44.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-04-30 08:03:44 on dist-test-slave-h4q8"
08:03:44.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:44.313179 23708 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster11388303748095761808/master-0/wal/instance:
08:03:44.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "91cc0cbb90c5488980c6a0da24388997"
08:03:44.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-04-30 08:03:44 on dist-test-slave-h4q8"
08:03:44.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:44.320755 23708 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.001s sys 0.004s
08:03:44.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:44.326740 23724 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:03:44.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:44.328047 23708 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.000s sys 0.004s
08:03:44.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:44.328433 23708 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster11388303748095761808/master-0/data,/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster11388303748095761808/master-0/wal
08:03:44.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "91cc0cbb90c5488980c6a0da24388997"
08:03:44.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-04-30 08:03:44 on dist-test-slave-h4q8"
08:03:44.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:44.328943 23708 fs_report.cc:389] FS layout report
08:03:44.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:03:44.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster11388303748095761808/master-0/wal
08:03:44.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster11388303748095761808/master-0/wal
08:03:44.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster11388303748095761808/master-0/data/data
08:03:44.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:03:44.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:03:44.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:03:44.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:03:44.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:03:44.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:03:44.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:03:44.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:03:44.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:03:44.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:03:44.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:03:44.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:44.401858 23708 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:03:44.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:44.403592 23708 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:03:44.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:44.404139 23708 kserver.cc:163] Server-wide thread pool size limit: 3276
08:03:44.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:44.473855 23708 rpc_server.cc:307] RPC server started. Bound to: 127.23.36.62:46005
08:03:44.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:44.473981 23775 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.23.36.62:46005 every 8 connection(s)
08:03:44.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:44.476979 23708 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster11388303748095761808/master-0/data/info.pb
08:03:44.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:44.477988 23696 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskf7pa4T/build/tsan/bin/kudu as pid 23708
08:03:44.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:44.478574 23696 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster11388303748095761808/master-0/wal/instance
08:03:44.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:44.483731 23696 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskf7pa4T/build/tsan/bin/kudu
08:03:44.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskf7pa4T/build/tsan/bin/kudu
08:03:44.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster11388303748095761808/master-1/wal
08:03:44.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster11388303748095761808/master-1/data
08:03:44.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:03:44.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:03:44.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:03:44.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:03:44.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:03:44.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:03:44.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster11388303748095761808/master-1/logs
08:03:44.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster11388303748095761808/master-1/data/info.pb
08:03:44.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:03:44.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:03:44.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:03:44.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:03:44.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:03:44.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:03:44.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:03:44.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:03:44.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:03:44.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:03:44.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:03:44.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:03:44.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.36.61:41985
08:03:44.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.36.61
08:03:44.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:03:44.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.36.20:36785
08:03:44.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:03:44.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:03:44.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:03:44.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:03:44.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.36.62:46005,127.23.36.61:41985,127.23.36.60:37459
08:03:44.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:03:44.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43625/jwks.json with env {}
08:03:44.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:44.483911 23776 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
08:03:44.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:44.502677 23776 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.36.62" port: 46005 } has no permanent_uuid. Determining permanent_uuid...
08:03:44.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:44.531271 23776 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.36.61" port: 41985 } has no permanent_uuid. Determining permanent_uuid...
08:03:44.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:03:44.534430 23727 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.23.36.61:41985: connect: Connection refused (error 111)
08:03:44.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:03:44.538726 23776 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.36.61:41985: Network error: Client connection negotiation failed: client connection to 127.23.36.61:41985: connect: Connection refused (error 111)
08:03:44.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:44.588297 23776 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.36.61" port: 41985 } attempt: 1
08:03:44.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:03:44.593308 23776 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.36.61:41985: Network error: Client connection negotiation failed: client connection to 127.23.36.61:41985: connect: Connection refused (error 111)
08:03:44.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:44.661693 23776 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.36.61" port: 41985 } attempt: 2
08:03:44.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:03:44.666247 23776 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.36.61:41985: Network error: Client connection negotiation failed: client connection to 127.23.36.61:41985: connect: Connection refused (error 111)
08:03:44.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:44.757650 23776 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.36.61" port: 41985 } attempt: 3
08:03:44.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:03:44.761999 23776 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.36.61:41985: Network error: Client connection negotiation failed: client connection to 127.23.36.61:41985: connect: Connection refused (error 111)
08:03:44.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:03:44.801195 23778 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
08:03:44.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:03:44.801990 23778 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:03:44.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:03:44.802775 23778 flags.cc:432] Enabled unsafe flag: --never_fsync=true
08:03:44.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:03:44.835835 23778 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
08:03:44.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:03:44.836231 23778 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
08:03:44.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:03:44.836599 23778 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
08:03:44.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:03:44.836941 23778 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
08:03:44.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:03:44.837316 23778 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
08:03:44.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:03:44.837607 23778 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:43625/jwks.json
08:03:44.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:44.875378 23778 master_runner.cc:386] Master server non-default flags:
08:03:44.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:03:44.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.36.20:36785
08:03:44.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:03:44.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:03:44.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster11388303748095761808/master-1/data
08:03:44.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster11388303748095761808/master-1/wal
08:03:44.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:03:44.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.36.62:46005,127.23.36.61:41985,127.23.36.60:37459
08:03:44.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:03:44.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:03:44.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:03:44.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.36.61:41985
08:03:44.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:03:44.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:03:44.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:03:44.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43625/jwks.json
08:03:44.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:03:44.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:03:44.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster11388303748095761808/master-1/data/info.pb
08:03:44.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.36.61
08:03:44.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:03:44.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:03:44.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:03:44.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:03:44.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:03:44.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:03:44.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster11388303748095761808/master-1/logs
08:03:44.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:03:44.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:03:44.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:03:44.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:03:44.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:03:44.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 8537ed5bb5e0e6ac1e481ff961e6b9d08cce7671
08:03:44.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:03:44.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 30 Apr 2026 07:43:22 UTC on e7f111948823
08:03:44.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11675
08:03:44.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:03:44.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:44.877142 23778 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:03:44.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:44.879565 23778 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:03:44.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:03:44.892490 23787 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:44.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:44.905452 23776 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.36.61" port: 41985 } attempt: 4
08:03:44.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:03:44.911209 23776 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.36.61:41985: Network error: Client connection negotiation failed: client connection to 127.23.36.61:41985: connect: Connection refused (error 111)
08:03:45.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:45.210726 23776 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.36.61" port: 41985 } attempt: 5
08:03:45.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:03:45.217959 23776 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.36.61:41985: Network error: Client connection negotiation failed: client connection to 127.23.36.61:41985: connect: Connection refused (error 111)
08:03:45.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:45.765616 23776 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.36.61" port: 41985 } attempt: 6
08:03:45.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:03:45.778097 23776 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.36.61:41985: Network error: Client connection negotiation failed: client connection to 127.23.36.61:41985: connect: Connection refused (error 111)
08:03:46.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:03:44.897097 23790 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:46.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:03:44.894227 23788 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:46.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:03:46.112458 23789 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1213 milliseconds
08:03:46.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:46.112582 23778 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
08:03:46.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:46.114228 23778 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:03:46.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:46.117614 23778 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:03:46.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:46.119465 23778 hybrid_clock.cc:648] HybridClock initialized: now 1777536226119469 us; error 88 us; skew 500 ppm
08:03:46.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:46.120288 23778 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:03:46.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:46.127101 23778 webserver.cc:492] Webserver started at http://127.23.36.61:39255/ using document root <none> and password file <none>
08:03:46.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:46.128185 23778 fs_manager.cc:362] Metadata directory not provided
08:03:46.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:46.128415 23778 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:03:46.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:46.128835 23778 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
08:03:46.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:46.133365 23778 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster11388303748095761808/master-1/data/instance:
08:03:46.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b53785e731cb44fd835d70e39bbe7a9c"
08:03:46.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-04-30 08:03:46 on dist-test-slave-h4q8"
08:03:46.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:46.134810 23778 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster11388303748095761808/master-1/wal/instance:
08:03:46.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b53785e731cb44fd835d70e39bbe7a9c"
08:03:46.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-04-30 08:03:46 on dist-test-slave-h4q8"
08:03:46.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:46.142318 23778 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.002s sys 0.005s
08:03:46.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:46.148011 23799 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:03:46.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:46.149191 23778 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.004s sys 0.002s
08:03:46.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:46.149480 23778 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster11388303748095761808/master-1/data,/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster11388303748095761808/master-1/wal
08:03:46.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b53785e731cb44fd835d70e39bbe7a9c"
08:03:46.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-04-30 08:03:46 on dist-test-slave-h4q8"
08:03:46.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:46.149912 23778 fs_report.cc:389] FS layout report
08:03:46.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:03:46.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster11388303748095761808/master-1/wal
08:03:46.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster11388303748095761808/master-1/wal
08:03:46.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster11388303748095761808/master-1/data/data
08:03:46.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:03:46.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:03:46.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:03:46.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:03:46.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:03:46.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:03:46.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:03:46.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:03:46.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:03:46.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:03:46.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:03:46.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:46.208956 23778 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:03:46.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:46.210474 23778 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:03:46.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:46.210879 23778 kserver.cc:163] Server-wide thread pool size limit: 3276
08:03:46.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:46.282176 23778 rpc_server.cc:307] RPC server started. Bound to: 127.23.36.61:41985
08:03:46.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:46.282249 23850 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.23.36.61:41985 every 8 connection(s)
08:03:46.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:46.285308 23778 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster11388303748095761808/master-1/data/info.pb
08:03:46.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:46.286389 23696 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskf7pa4T/build/tsan/bin/kudu as pid 23778
08:03:46.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:46.286866 23696 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster11388303748095761808/master-1/wal/instance
08:03:46.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:46.289759 23696 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskf7pa4T/build/tsan/bin/kudu
08:03:46.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskf7pa4T/build/tsan/bin/kudu
08:03:46.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster11388303748095761808/master-2/wal
08:03:46.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster11388303748095761808/master-2/data
08:03:46.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:03:46.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:03:46.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:03:46.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:03:46.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:03:46.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:03:46.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster11388303748095761808/master-2/logs
08:03:46.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster11388303748095761808/master-2/data/info.pb
08:03:46.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:03:46.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:03:46.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:03:46.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:03:46.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:03:46.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:03:46.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:03:46.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:03:46.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:03:46.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:03:46.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:03:46.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:03:46.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.36.60:37459
08:03:46.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.36.60
08:03:46.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:03:46.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.36.20:36785
08:03:46.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:03:46.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:03:46.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:03:46.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:03:46.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.36.62:46005,127.23.36.61:41985,127.23.36.60:37459
08:03:46.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:03:46.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43625/jwks.json with env {}
08:03:46.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:46.294603 23851 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
08:03:46.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:46.310633 23851 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.36.62" port: 46005 } has no permanent_uuid. Determining permanent_uuid...
08:03:46.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:46.332021 23851 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.36.61" port: 41985 } has no permanent_uuid. Determining permanent_uuid...
08:03:46.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:46.346247 23851 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.36.60" port: 37459 } has no permanent_uuid. Determining permanent_uuid...
08:03:46.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:03:46.348853 23800 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.23.36.60:37459: connect: Connection refused (error 111)
08:03:46.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:03:46.353292 23851 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.36.60:37459: Network error: Client connection negotiation failed: client connection to 127.23.36.60:37459: connect: Connection refused (error 111)
08:03:46.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:46.402787 23851 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.36.60" port: 37459 } attempt: 1
08:03:46.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:03:46.407608 23851 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.36.60:37459: Network error: Client connection negotiation failed: client connection to 127.23.36.60:37459: connect: Connection refused (error 111)
08:03:46.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:46.476136 23851 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.36.60" port: 37459 } attempt: 2
08:03:46.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:03:46.481191 23851 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.36.60:37459: Network error: Client connection negotiation failed: client connection to 127.23.36.60:37459: connect: Connection refused (error 111)
08:03:46.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:46.572716 23851 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.36.60" port: 37459 } attempt: 3
08:03:46.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:03:46.578449 23851 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.36.60:37459: Network error: Client connection negotiation failed: client connection to 127.23.36.60:37459: connect: Connection refused (error 111)
08:03:46.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:03:46.614892 23853 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
08:03:46.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:03:46.615628 23853 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:03:46.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:03:46.616314 23853 flags.cc:432] Enabled unsafe flag: --never_fsync=true
08:03:46.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:03:46.649111 23853 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
08:03:46.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:03:46.649514 23853 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
08:03:46.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:03:46.649861 23853 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
08:03:46.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:03:46.650219 23853 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
08:03:46.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:03:46.650621 23853 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
08:03:46.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:03:46.650887 23853 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:43625/jwks.json
08:03:46.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:46.688122 23853 master_runner.cc:386] Master server non-default flags:
08:03:46.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:03:46.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.36.20:36785
08:03:46.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:03:46.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:03:46.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster11388303748095761808/master-2/data
08:03:46.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster11388303748095761808/master-2/wal
08:03:46.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:03:46.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.36.62:46005,127.23.36.61:41985,127.23.36.60:37459
08:03:46.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:03:46.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:03:46.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:03:46.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.36.60:37459
08:03:46.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:03:46.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:03:46.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:03:46.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43625/jwks.json
08:03:46.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:03:46.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:03:46.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster11388303748095761808/master-2/data/info.pb
08:03:46.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.36.60
08:03:46.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:03:46.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:03:46.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:03:46.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:03:46.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:03:46.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:03:46.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster11388303748095761808/master-2/logs
08:03:46.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:03:46.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:03:46.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:03:46.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:03:46.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:03:46.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 8537ed5bb5e0e6ac1e481ff961e6b9d08cce7671
08:03:46.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:03:46.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 30 Apr 2026 07:43:22 UTC on e7f111948823
08:03:46.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11675
08:03:46.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:03:46.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:46.689927 23853 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:03:46.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:46.692296 23853 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:03:46.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:03:46.704473 23863 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:46.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:46.721993 23851 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.36.60" port: 37459 } attempt: 4
08:03:46.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:03:46.729286 23851 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.36.60:37459: Network error: Client connection negotiation failed: client connection to 127.23.36.60:37459: connect: Connection refused (error 111)
08:03:46.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:46.838750 23776 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.36.61" port: 41985 } attempt: 7
08:03:46.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:46.873957 23776 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.36.60" port: 37459 } has no permanent_uuid. Determining permanent_uuid...
08:03:46.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:03:46.887521 23776 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.36.60:37459: Network error: Client connection negotiation failed: client connection to 127.23.36.60:37459: connect: Connection refused (error 111)
08:03:46.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:46.946153 23776 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.36.60" port: 37459 } attempt: 1
08:03:46.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:03:46.955243 23776 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.36.60:37459: Network error: Client connection negotiation failed: client connection to 127.23.36.60:37459: connect: Connection refused (error 111)
08:03:47.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:47.028869 23851 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.36.60" port: 37459 } attempt: 5
08:03:47.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:47.037842 23776 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.36.60" port: 37459 } attempt: 2
08:03:47.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:03:47.045703 23851 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.36.60:37459: Network error: Client connection negotiation failed: client connection to 127.23.36.60:37459: connect: Connection refused (error 111)
08:03:47.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:03:47.062618 23776 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.36.60:37459: Network error: Client connection negotiation failed: client connection to 127.23.36.60:37459: connect: Connection refused (error 111)
08:03:47.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:47.148259 23776 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.36.60" port: 37459 } attempt: 3
08:03:47.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:03:47.156936 23776 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.36.60:37459: Network error: Client connection negotiation failed: client connection to 127.23.36.60:37459: connect: Connection refused (error 111)
08:03:47.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:47.297514 23776 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.36.60" port: 37459 } attempt: 4
08:03:47.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:03:47.305496 23776 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.36.60:37459: Network error: Client connection negotiation failed: client connection to 127.23.36.60:37459: connect: Connection refused (error 111)
08:03:47.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:47.589109 23776 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.36.60" port: 37459 } attempt: 5
08:03:47.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:47.593364 23851 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.36.60" port: 37459 } attempt: 6
08:03:47.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:03:47.600188 23776 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.36.60:37459: Network error: Client connection negotiation failed: client connection to 127.23.36.60:37459: connect: Connection refused (error 111)
08:03:47.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:03:47.619709 23851 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.36.60:37459: Network error: Client connection negotiation failed: client connection to 127.23.36.60:37459: connect: Connection refused (error 111)
08:03:48.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:48.152756 23776 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.36.60" port: 37459 } attempt: 6
08:03:48.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:03:48.181951 23776 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.36.60:37459: Network error: Client connection negotiation failed: client connection to 127.23.36.60:37459: connect: Connection refused (error 111)
08:03:48.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:03:48.112044 23862 debug-util.cc:398] Leaking SignalData structure 0x7b0800034cc0 after lost signal to thread 23853
08:03:48.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:03:48.334625 23862 kernel_stack_watchdog.cc:198] Thread 23853 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 401ms:
08:03:48.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
08:03:48.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
08:03:48.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
08:03:48.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
08:03:48.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:03:46.706616 23864 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:48.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:03:48.339566 23853 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.634s user 0.638s sys 0.983s
08:03:48.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:03:48.339956 23865 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1628 milliseconds
08:03:48.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:03:48.339977 23853 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.635s user 0.638s sys 0.983s
08:03:48.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:03:48.341405 23866 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:48.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:48.341476 23853 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
08:03:48.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:48.342614 23853 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:03:48.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:48.344965 23853 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:03:48.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:48.346338 23853 hybrid_clock.cc:648] HybridClock initialized: now 1777536228346321 us; error 52 us; skew 500 ppm
08:03:48.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:48.347178 23853 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:03:48.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:48.353107 23853 webserver.cc:492] Webserver started at http://127.23.36.60:41955/ using document root <none> and password file <none>
08:03:48.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:48.354043 23853 fs_manager.cc:362] Metadata directory not provided
08:03:48.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:48.354230 23853 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:03:48.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:48.354784 23853 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
08:03:48.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:48.359488 23853 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster11388303748095761808/master-2/data/instance:
08:03:48.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d2bf70afa9c24023b22b92050d87c195"
08:03:48.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-04-30 08:03:48 on dist-test-slave-h4q8"
08:03:48.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:48.360520 23853 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster11388303748095761808/master-2/wal/instance:
08:03:48.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d2bf70afa9c24023b22b92050d87c195"
08:03:48.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-04-30 08:03:48 on dist-test-slave-h4q8"
08:03:48.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:48.368060 23853 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.007s sys 0.003s
08:03:48.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:48.373584 23880 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:03:48.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:48.374740 23853 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.001s
08:03:48.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:48.375010 23853 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster11388303748095761808/master-2/data,/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster11388303748095761808/master-2/wal
08:03:48.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d2bf70afa9c24023b22b92050d87c195"
08:03:48.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-04-30 08:03:48 on dist-test-slave-h4q8"
08:03:48.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:48.375310 23853 fs_report.cc:389] FS layout report
08:03:48.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:03:48.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster11388303748095761808/master-2/wal
08:03:48.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster11388303748095761808/master-2/wal
08:03:48.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster11388303748095761808/master-2/data/data
08:03:48.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:03:48.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:03:48.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:03:48.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:03:48.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:03:48.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:03:48.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:03:48.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:03:48.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:03:48.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:03:48.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:03:48.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:48.439432 23853 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:03:48.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:48.441075 23853 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:03:48.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:48.441484 23853 kserver.cc:163] Server-wide thread pool size limit: 3276
08:03:48.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:48.514822 23853 rpc_server.cc:307] RPC server started. Bound to: 127.23.36.60:37459
08:03:48.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:48.514916 23931 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.23.36.60:37459 every 8 connection(s)
08:03:48.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:48.517719 23853 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster11388303748095761808/master-2/data/info.pb
08:03:48.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:48.520826 23696 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskf7pa4T/build/tsan/bin/kudu as pid 23853
08:03:48.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:48.521301 23696 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster11388303748095761808/master-2/wal/instance
08:03:48.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:48.525373 23932 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
08:03:48.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:48.539629 23932 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.36.62" port: 46005 } has no permanent_uuid. Determining permanent_uuid...
08:03:48.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:48.563516 23932 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.36.61" port: 41985 } has no permanent_uuid. Determining permanent_uuid...
08:03:48.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:48.574215 23932 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.36.60" port: 37459 } has no permanent_uuid. Determining permanent_uuid...
08:03:48.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:48.594329 23932 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d2bf70afa9c24023b22b92050d87c195: Bootstrap starting.
08:03:48.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:48.601313 23932 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P d2bf70afa9c24023b22b92050d87c195: Neither blocks nor log segments found. Creating new log.
08:03:48.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:48.603307 23932 log.cc:826] T 00000000000000000000000000000000 P d2bf70afa9c24023b22b92050d87c195: Log is configured to *not* fsync() on all Append() calls
08:03:48.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:48.607981 23932 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d2bf70afa9c24023b22b92050d87c195: No bootstrap required, opened a new log
08:03:48.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:48.627979 23932 raft_consensus.cc:359] T 00000000000000000000000000000000 P d2bf70afa9c24023b22b92050d87c195 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "91cc0cbb90c5488980c6a0da24388997" member_type: VOTER last_known_addr { host: "127.23.36.62" port: 46005 } } peers { permanent_uuid: "b53785e731cb44fd835d70e39bbe7a9c" member_type: VOTER last_known_addr { host: "127.23.36.61" port: 41985 } } peers { permanent_uuid: "d2bf70afa9c24023b22b92050d87c195" member_type: VOTER last_known_addr { host: "127.23.36.60" port: 37459 } }
08:03:48.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:48.628698 23932 raft_consensus.cc:385] T 00000000000000000000000000000000 P d2bf70afa9c24023b22b92050d87c195 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:03:48.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:48.628989 23932 raft_consensus.cc:740] T 00000000000000000000000000000000 P d2bf70afa9c24023b22b92050d87c195 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d2bf70afa9c24023b22b92050d87c195, State: Initialized, Role: FOLLOWER
08:03:48.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:48.629976 23932 consensus_queue.cc:260] T 00000000000000000000000000000000 P d2bf70afa9c24023b22b92050d87c195 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "91cc0cbb90c5488980c6a0da24388997" member_type: VOTER last_known_addr { host: "127.23.36.62" port: 46005 } } peers { permanent_uuid: "b53785e731cb44fd835d70e39bbe7a9c" member_type: VOTER last_known_addr { host: "127.23.36.61" port: 41985 } } peers { permanent_uuid: "d2bf70afa9c24023b22b92050d87c195" member_type: VOTER last_known_addr { host: "127.23.36.60" port: 37459 } }
08:03:48.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:48.632483 23941 sys_catalog.cc:455] T 00000000000000000000000000000000 P d2bf70afa9c24023b22b92050d87c195 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "91cc0cbb90c5488980c6a0da24388997" member_type: VOTER last_known_addr { host: "127.23.36.62" port: 46005 } } peers { permanent_uuid: "b53785e731cb44fd835d70e39bbe7a9c" member_type: VOTER last_known_addr { host: "127.23.36.61" port: 41985 } } peers { permanent_uuid: "d2bf70afa9c24023b22b92050d87c195" member_type: VOTER last_known_addr { host: "127.23.36.60" port: 37459 } } }
08:03:48.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:48.633813 23941 sys_catalog.cc:458] T 00000000000000000000000000000000 P d2bf70afa9c24023b22b92050d87c195 [sys.catalog]: This master's current role is: FOLLOWER
08:03:48.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:48.634509 23932 sys_catalog.cc:565] T 00000000000000000000000000000000 P d2bf70afa9c24023b22b92050d87c195 [sys.catalog]: configured and running, proceeding with master startup.
08:03:48.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:03:48.657976 23952 catalog_manager.cc:1568] T 00000000000000000000000000000000 P d2bf70afa9c24023b22b92050d87c195: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:03:48.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:03:48.658298 23952 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:03:48.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:48.680378 23851 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.36.60" port: 37459 } attempt: 7
08:03:48.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:48.654650 23853 master_runner.cc:186] Error getting master registration for 127.23.36.62:46005: OK, instance_id { permanent_uuid: "91cc0cbb90c5488980c6a0da24388997" instance_seqno: 1777536224455488 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
08:03:48.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:48.686259 23853 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
08:03:48.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:03:48.696614 23886 tablet.cc:2404] T 00000000000000000000000000000000 P d2bf70afa9c24023b22b92050d87c195: Can't schedule compaction. Clean time has not been advanced past its initial value.
08:03:48.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:48.704181 23941 raft_consensus.cc:493] T 00000000000000000000000000000000 P d2bf70afa9c24023b22b92050d87c195 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:03:48.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:48.704914 23851 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b53785e731cb44fd835d70e39bbe7a9c: Bootstrap starting.
08:03:48.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:48.704710 23941 raft_consensus.cc:515] T 00000000000000000000000000000000 P d2bf70afa9c24023b22b92050d87c195 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "91cc0cbb90c5488980c6a0da24388997" member_type: VOTER last_known_addr { host: "127.23.36.62" port: 46005 } } peers { permanent_uuid: "b53785e731cb44fd835d70e39bbe7a9c" member_type: VOTER last_known_addr { host: "127.23.36.61" port: 41985 } } peers { permanent_uuid: "d2bf70afa9c24023b22b92050d87c195" member_type: VOTER last_known_addr { host: "127.23.36.60" port: 37459 } }
08:03:48.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:48.708417 23941 leader_election.cc:290] T 00000000000000000000000000000000 P d2bf70afa9c24023b22b92050d87c195 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 91cc0cbb90c5488980c6a0da24388997 (127.23.36.62:46005), b53785e731cb44fd835d70e39bbe7a9c (127.23.36.61:41985)
08:03:48.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:48.713658 23851 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P b53785e731cb44fd835d70e39bbe7a9c: Neither blocks nor log segments found. Creating new log.
08:03:48.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:48.717150 23851 log.cc:826] T 00000000000000000000000000000000 P b53785e731cb44fd835d70e39bbe7a9c: Log is configured to *not* fsync() on all Append() calls
08:03:48.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:48.731664 23851 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b53785e731cb44fd835d70e39bbe7a9c: No bootstrap required, opened a new log
08:03:48.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:48.766081 23851 raft_consensus.cc:359] T 00000000000000000000000000000000 P b53785e731cb44fd835d70e39bbe7a9c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "91cc0cbb90c5488980c6a0da24388997" member_type: VOTER last_known_addr { host: "127.23.36.62" port: 46005 } } peers { permanent_uuid: "b53785e731cb44fd835d70e39bbe7a9c" member_type: VOTER last_known_addr { host: "127.23.36.61" port: 41985 } } peers { permanent_uuid: "d2bf70afa9c24023b22b92050d87c195" member_type: VOTER last_known_addr { host: "127.23.36.60" port: 37459 } }
08:03:48.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:48.767603 23851 raft_consensus.cc:385] T 00000000000000000000000000000000 P b53785e731cb44fd835d70e39bbe7a9c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:03:48.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:48.768258 23851 raft_consensus.cc:740] T 00000000000000000000000000000000 P b53785e731cb44fd835d70e39bbe7a9c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: b53785e731cb44fd835d70e39bbe7a9c, State: Initialized, Role: FOLLOWER
08:03:48.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:48.769621 23851 consensus_queue.cc:260] T 00000000000000000000000000000000 P b53785e731cb44fd835d70e39bbe7a9c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "91cc0cbb90c5488980c6a0da24388997" member_type: VOTER last_known_addr { host: "127.23.36.62" port: 46005 } } peers { permanent_uuid: "b53785e731cb44fd835d70e39bbe7a9c" member_type: VOTER last_known_addr { host: "127.23.36.61" port: 41985 } } peers { permanent_uuid: "d2bf70afa9c24023b22b92050d87c195" member_type: VOTER last_known_addr { host: "127.23.36.60" port: 37459 } }
08:03:48.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:48.772749 23956 sys_catalog.cc:455] T 00000000000000000000000000000000 P b53785e731cb44fd835d70e39bbe7a9c [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "91cc0cbb90c5488980c6a0da24388997" member_type: VOTER last_known_addr { host: "127.23.36.62" port: 46005 } } peers { permanent_uuid: "b53785e731cb44fd835d70e39bbe7a9c" member_type: VOTER last_known_addr { host: "127.23.36.61" port: 41985 } } peers { permanent_uuid: "d2bf70afa9c24023b22b92050d87c195" member_type: VOTER last_known_addr { host: "127.23.36.60" port: 37459 } } }
08:03:48.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:48.773561 23956 sys_catalog.cc:458] T 00000000000000000000000000000000 P b53785e731cb44fd835d70e39bbe7a9c [sys.catalog]: This master's current role is: FOLLOWER
08:03:48.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:48.775456 23851 sys_catalog.cc:565] T 00000000000000000000000000000000 P b53785e731cb44fd835d70e39bbe7a9c [sys.catalog]: configured and running, proceeding with master startup.
08:03:48.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:48.777541 23826 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "d2bf70afa9c24023b22b92050d87c195" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "b53785e731cb44fd835d70e39bbe7a9c" is_pre_election: true
08:03:48.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:48.778513 23826 raft_consensus.cc:2468] T 00000000000000000000000000000000 P b53785e731cb44fd835d70e39bbe7a9c [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate d2bf70afa9c24023b22b92050d87c195 in term 0.
08:03:48.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:48.780325 23883 leader_election.cc:304] T 00000000000000000000000000000000 P d2bf70afa9c24023b22b92050d87c195 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: b53785e731cb44fd835d70e39bbe7a9c, d2bf70afa9c24023b22b92050d87c195; no voters:
08:03:48.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:48.781854 23941 raft_consensus.cc:2804] T 00000000000000000000000000000000 P d2bf70afa9c24023b22b92050d87c195 [term 0 FOLLOWER]: Leader pre-election won for term 1
08:03:48.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:48.782371 23941 raft_consensus.cc:493] T 00000000000000000000000000000000 P d2bf70afa9c24023b22b92050d87c195 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
08:03:48.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:48.782716 23941 raft_consensus.cc:3060] T 00000000000000000000000000000000 P d2bf70afa9c24023b22b92050d87c195 [term 0 FOLLOWER]: Advancing to term 1
08:03:48.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:48.792562 23941 raft_consensus.cc:515] T 00000000000000000000000000000000 P d2bf70afa9c24023b22b92050d87c195 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "91cc0cbb90c5488980c6a0da24388997" member_type: VOTER last_known_addr { host: "127.23.36.62" port: 46005 } } peers { permanent_uuid: "b53785e731cb44fd835d70e39bbe7a9c" member_type: VOTER last_known_addr { host: "127.23.36.61" port: 41985 } } peers { permanent_uuid: "d2bf70afa9c24023b22b92050d87c195" member_type: VOTER last_known_addr { host: "127.23.36.60" port: 37459 } }
08:03:48.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:48.796188 23826 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "d2bf70afa9c24023b22b92050d87c195" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "b53785e731cb44fd835d70e39bbe7a9c"
08:03:48.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:48.798622 23826 raft_consensus.cc:3060] T 00000000000000000000000000000000 P b53785e731cb44fd835d70e39bbe7a9c [term 0 FOLLOWER]: Advancing to term 1
08:03:48.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:48.803725 23941 leader_election.cc:290] T 00000000000000000000000000000000 P d2bf70afa9c24023b22b92050d87c195 [CANDIDATE]: Term 1 election: Requested vote from peers 91cc0cbb90c5488980c6a0da24388997 (127.23.36.62:46005), b53785e731cb44fd835d70e39bbe7a9c (127.23.36.61:41985)
08:03:48.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:48.809247 23826 raft_consensus.cc:2468] T 00000000000000000000000000000000 P b53785e731cb44fd835d70e39bbe7a9c [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate d2bf70afa9c24023b22b92050d87c195 in term 1.
08:03:48.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:48.810454 23883 leader_election.cc:304] T 00000000000000000000000000000000 P d2bf70afa9c24023b22b92050d87c195 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: b53785e731cb44fd835d70e39bbe7a9c, d2bf70afa9c24023b22b92050d87c195; no voters:
08:03:48.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:48.811406 23941 raft_consensus.cc:2804] T 00000000000000000000000000000000 P d2bf70afa9c24023b22b92050d87c195 [term 1 FOLLOWER]: Leader election won for term 1
08:03:48.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:48.813632 23941 raft_consensus.cc:697] T 00000000000000000000000000000000 P d2bf70afa9c24023b22b92050d87c195 [term 1 LEADER]: Becoming Leader. State: Replica: d2bf70afa9c24023b22b92050d87c195, State: Running, Role: LEADER
08:03:48.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:48.814687 23941 consensus_queue.cc:237] T 00000000000000000000000000000000 P d2bf70afa9c24023b22b92050d87c195 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "91cc0cbb90c5488980c6a0da24388997" member_type: VOTER last_known_addr { host: "127.23.36.62" port: 46005 } } peers { permanent_uuid: "b53785e731cb44fd835d70e39bbe7a9c" member_type: VOTER last_known_addr { host: "127.23.36.61" port: 41985 } } peers { permanent_uuid: "d2bf70afa9c24023b22b92050d87c195" member_type: VOTER last_known_addr { host: "127.23.36.60" port: 37459 } }
08:03:48.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:48.826638 23963 sys_catalog.cc:455] T 00000000000000000000000000000000 P d2bf70afa9c24023b22b92050d87c195 [sys.catalog]: SysCatalogTable state changed. Reason: New leader d2bf70afa9c24023b22b92050d87c195. Latest consensus state: current_term: 1 leader_uuid: "d2bf70afa9c24023b22b92050d87c195" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "91cc0cbb90c5488980c6a0da24388997" member_type: VOTER last_known_addr { host: "127.23.36.62" port: 46005 } } peers { permanent_uuid: "b53785e731cb44fd835d70e39bbe7a9c" member_type: VOTER last_known_addr { host: "127.23.36.61" port: 41985 } } peers { permanent_uuid: "d2bf70afa9c24023b22b92050d87c195" member_type: VOTER last_known_addr { host: "127.23.36.60" port: 37459 } } }
08:03:48.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:48.835251 23963 sys_catalog.cc:458] T 00000000000000000000000000000000 P d2bf70afa9c24023b22b92050d87c195 [sys.catalog]: This master's current role is: LEADER
08:03:48.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:48.839893 23968 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
08:03:48.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:48.848310 23968 catalog_manager.cc:1494] Initializing Kudu cluster ID...
08:03:48.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:48.852922 23970 catalog_manager.cc:805] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
08:03:48.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:48.869361 23826 raft_consensus.cc:1275] T 00000000000000000000000000000000 P b53785e731cb44fd835d70e39bbe7a9c [term 1 FOLLOWER]: Refusing update from remote peer d2bf70afa9c24023b22b92050d87c195: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
08:03:48.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:48.871275 23963 consensus_queue.cc:1048] T 00000000000000000000000000000000 P d2bf70afa9c24023b22b92050d87c195 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b53785e731cb44fd835d70e39bbe7a9c" member_type: VOTER last_known_addr { host: "127.23.36.61" port: 41985 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
08:03:48.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:48.858947 23778 master_runner.cc:186] Error getting master registration for 127.23.36.62:46005: OK, instance_id { permanent_uuid: "91cc0cbb90c5488980c6a0da24388997" instance_seqno: 1777536224455488 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
08:03:48.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:48.894091 23778 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
08:03:48.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:48.907621 23971 mvcc.cc:204] Tried to move back new op lower bound from 7280788393429913600 to 7280788393233371136. Current Snapshot: MvccSnapshot[applied={T|T < 7280788393429913600}]
08:03:48.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:48.905324 23956 sys_catalog.cc:455] T 00000000000000000000000000000000 P b53785e731cb44fd835d70e39bbe7a9c [sys.catalog]: SysCatalogTable state changed. Reason: New leader d2bf70afa9c24023b22b92050d87c195. Latest consensus state: current_term: 1 leader_uuid: "d2bf70afa9c24023b22b92050d87c195" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "91cc0cbb90c5488980c6a0da24388997" member_type: VOTER last_known_addr { host: "127.23.36.62" port: 46005 } } peers { permanent_uuid: "b53785e731cb44fd835d70e39bbe7a9c" member_type: VOTER last_known_addr { host: "127.23.36.61" port: 41985 } } peers { permanent_uuid: "d2bf70afa9c24023b22b92050d87c195" member_type: VOTER last_known_addr { host: "127.23.36.60" port: 37459 } } }
08:03:48.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:48.909862 23941 sys_catalog.cc:455] T 00000000000000000000000000000000 P d2bf70afa9c24023b22b92050d87c195 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "d2bf70afa9c24023b22b92050d87c195" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "91cc0cbb90c5488980c6a0da24388997" member_type: VOTER last_known_addr { host: "127.23.36.62" port: 46005 } } peers { permanent_uuid: "b53785e731cb44fd835d70e39bbe7a9c" member_type: VOTER last_known_addr { host: "127.23.36.61" port: 41985 } } peers { permanent_uuid: "d2bf70afa9c24023b22b92050d87c195" member_type: VOTER last_known_addr { host: "127.23.36.60" port: 37459 } } }
08:03:48.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:48.910746 23941 sys_catalog.cc:458] T 00000000000000000000000000000000 P d2bf70afa9c24023b22b92050d87c195 [sys.catalog]: This master's current role is: LEADER
08:03:48.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:48.910455 23956 sys_catalog.cc:458] T 00000000000000000000000000000000 P b53785e731cb44fd835d70e39bbe7a9c [sys.catalog]: This master's current role is: FOLLOWER
08:03:48.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:48.922006 23968 catalog_manager.cc:1357] Generated new cluster ID: ad2e883dd84f401982a0078b10967bc7
08:03:48.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:48.922430 23968 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
08:03:48.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:48.940974 23956 sys_catalog.cc:455] T 00000000000000000000000000000000 P b53785e731cb44fd835d70e39bbe7a9c [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "d2bf70afa9c24023b22b92050d87c195" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "91cc0cbb90c5488980c6a0da24388997" member_type: VOTER last_known_addr { host: "127.23.36.62" port: 46005 } } peers { permanent_uuid: "b53785e731cb44fd835d70e39bbe7a9c" member_type: VOTER last_known_addr { host: "127.23.36.61" port: 41985 } } peers { permanent_uuid: "d2bf70afa9c24023b22b92050d87c195" member_type: VOTER last_known_addr { host: "127.23.36.60" port: 37459 } } }
08:03:48.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:48.944506 23956 sys_catalog.cc:458] T 00000000000000000000000000000000 P b53785e731cb44fd835d70e39bbe7a9c [sys.catalog]: This master's current role is: FOLLOWER
08:03:48.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:48.964833 23968 catalog_manager.cc:1380] Generated new certificate authority record
08:03:48.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:48.969842 23968 catalog_manager.cc:1514] Loading token signing keys...
08:03:49.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:49.007929 23968 catalog_manager.cc:6044] T 00000000000000000000000000000000 P d2bf70afa9c24023b22b92050d87c195: Generated new TSK 0
08:03:49.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:49.010450 23968 catalog_manager.cc:1524] Initializing in-progress tserver states...
08:03:49.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:49.040205 23696 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskf7pa4T/build/tsan/bin/kudu
08:03:49.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskf7pa4T/build/tsan/bin/kudu
08:03:49.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster11388303748095761808/ts-0/wal
08:03:49.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster11388303748095761808/ts-0/data
08:03:49.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:03:49.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:03:49.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:03:49.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:03:49.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:03:49.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:03:49.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster11388303748095761808/ts-0/logs
08:03:49.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster11388303748095761808/ts-0/data/info.pb
08:03:49.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:03:49.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:03:49.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:03:49.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:03:49.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:03:49.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:03:49.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:03:49.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:03:49.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
08:03:49.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:03:49.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.36.1:0
08:03:49.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.36.1
08:03:49.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.36.1
08:03:49.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:03:49.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.36.62:46005,127.23.36.61:41985,127.23.36.60:37459
08:03:49.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.36.20:36785
08:03:49.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:03:49.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:03:49.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
08:03:49.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:49.215495 23776 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.36.60" port: 37459 } attempt: 7
08:03:49.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:49.253122 23776 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 91cc0cbb90c5488980c6a0da24388997: Bootstrap starting.
08:03:49.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:49.260785 23776 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 91cc0cbb90c5488980c6a0da24388997: Neither blocks nor log segments found. Creating new log.
08:03:49.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:49.263178 23776 log.cc:826] T 00000000000000000000000000000000 P 91cc0cbb90c5488980c6a0da24388997: Log is configured to *not* fsync() on all Append() calls
08:03:49.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:49.269830 23776 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 91cc0cbb90c5488980c6a0da24388997: No bootstrap required, opened a new log
08:03:49.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:49.298652 23776 raft_consensus.cc:359] T 00000000000000000000000000000000 P 91cc0cbb90c5488980c6a0da24388997 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "91cc0cbb90c5488980c6a0da24388997" member_type: VOTER last_known_addr { host: "127.23.36.62" port: 46005 } } peers { permanent_uuid: "b53785e731cb44fd835d70e39bbe7a9c" member_type: VOTER last_known_addr { host: "127.23.36.61" port: 41985 } } peers { permanent_uuid: "d2bf70afa9c24023b22b92050d87c195" member_type: VOTER last_known_addr { host: "127.23.36.60" port: 37459 } }
08:03:49.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:49.299521 23776 raft_consensus.cc:385] T 00000000000000000000000000000000 P 91cc0cbb90c5488980c6a0da24388997 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:03:49.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:49.299825 23776 raft_consensus.cc:740] T 00000000000000000000000000000000 P 91cc0cbb90c5488980c6a0da24388997 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 91cc0cbb90c5488980c6a0da24388997, State: Initialized, Role: FOLLOWER
08:03:49.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:49.300994 23776 consensus_queue.cc:260] T 00000000000000000000000000000000 P 91cc0cbb90c5488980c6a0da24388997 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "91cc0cbb90c5488980c6a0da24388997" member_type: VOTER last_known_addr { host: "127.23.36.62" port: 46005 } } peers { permanent_uuid: "b53785e731cb44fd835d70e39bbe7a9c" member_type: VOTER last_known_addr { host: "127.23.36.61" port: 41985 } } peers { permanent_uuid: "d2bf70afa9c24023b22b92050d87c195" member_type: VOTER last_known_addr { host: "127.23.36.60" port: 37459 } }
08:03:49.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:49.305578 23776 sys_catalog.cc:565] T 00000000000000000000000000000000 P 91cc0cbb90c5488980c6a0da24388997 [sys.catalog]: configured and running, proceeding with master startup.
08:03:49.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:49.306049 23750 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "d2bf70afa9c24023b22b92050d87c195" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "91cc0cbb90c5488980c6a0da24388997"
08:03:49.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:49.306697 23751 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "d2bf70afa9c24023b22b92050d87c195" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "91cc0cbb90c5488980c6a0da24388997" is_pre_election: true
08:03:49.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:49.307533 23749 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 91cc0cbb90c5488980c6a0da24388997 [term 0 FOLLOWER]: Advancing to term 1
08:03:49.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:49.304631 23981 sys_catalog.cc:455] T 00000000000000000000000000000000 P 91cc0cbb90c5488980c6a0da24388997 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "91cc0cbb90c5488980c6a0da24388997" member_type: VOTER last_known_addr { host: "127.23.36.62" port: 46005 } } peers { permanent_uuid: "b53785e731cb44fd835d70e39bbe7a9c" member_type: VOTER last_known_addr { host: "127.23.36.61" port: 41985 } } peers { permanent_uuid: "d2bf70afa9c24023b22b92050d87c195" member_type: VOTER last_known_addr { host: "127.23.36.60" port: 37459 } } }
08:03:49.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:49.319546 23981 sys_catalog.cc:458] T 00000000000000000000000000000000 P 91cc0cbb90c5488980c6a0da24388997 [sys.catalog]: This master's current role is: FOLLOWER
08:03:49.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:49.325574 23749 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 91cc0cbb90c5488980c6a0da24388997 [term 1 FOLLOWER]: Refusing update from remote peer d2bf70afa9c24023b22b92050d87c195: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
08:03:49.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:49.329684 23963 consensus_queue.cc:1048] T 00000000000000000000000000000000 P d2bf70afa9c24023b22b92050d87c195 [LEADER]: Connected to new peer: Peer: permanent_uuid: "91cc0cbb90c5488980c6a0da24388997" member_type: VOTER last_known_addr { host: "127.23.36.62" port: 46005 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
08:03:49.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:49.412866 23981 sys_catalog.cc:455] T 00000000000000000000000000000000 P 91cc0cbb90c5488980c6a0da24388997 [sys.catalog]: SysCatalogTable state changed. Reason: New leader d2bf70afa9c24023b22b92050d87c195. Latest consensus state: current_term: 1 leader_uuid: "d2bf70afa9c24023b22b92050d87c195" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "91cc0cbb90c5488980c6a0da24388997" member_type: VOTER last_known_addr { host: "127.23.36.62" port: 46005 } } peers { permanent_uuid: "b53785e731cb44fd835d70e39bbe7a9c" member_type: VOTER last_known_addr { host: "127.23.36.61" port: 41985 } } peers { permanent_uuid: "d2bf70afa9c24023b22b92050d87c195" member_type: VOTER last_known_addr { host: "127.23.36.60" port: 37459 } } }
08:03:49.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:49.413050 23988 sys_catalog.cc:455] T 00000000000000000000000000000000 P 91cc0cbb90c5488980c6a0da24388997 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "d2bf70afa9c24023b22b92050d87c195" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "91cc0cbb90c5488980c6a0da24388997" member_type: VOTER last_known_addr { host: "127.23.36.62" port: 46005 } } peers { permanent_uuid: "b53785e731cb44fd835d70e39bbe7a9c" member_type: VOTER last_known_addr { host: "127.23.36.61" port: 41985 } } peers { permanent_uuid: "d2bf70afa9c24023b22b92050d87c195" member_type: VOTER last_known_addr { host: "127.23.36.60" port: 37459 } } }
08:03:49.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:49.414260 23981 sys_catalog.cc:458] T 00000000000000000000000000000000 P 91cc0cbb90c5488980c6a0da24388997 [sys.catalog]: This master's current role is: FOLLOWER
08:03:49.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:49.414588 23988 sys_catalog.cc:458] T 00000000000000000000000000000000 P 91cc0cbb90c5488980c6a0da24388997 [sys.catalog]: This master's current role is: FOLLOWER
08:03:49.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:49.427104 23998 catalog_manager.cc:805] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
08:03:49.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:03:49.509505 23977 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
08:03:49.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:03:49.510339 23977 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:03:49.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:03:49.511139 23977 flags.cc:432] Enabled unsafe flag: --never_fsync=true
08:03:49.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:03:49.579538 23977 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
08:03:49.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:03:49.581311 23977 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.23.36.1
08:03:49.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:49.629212 23977 tablet_server_runner.cc:78] Tablet server non-default flags:
08:03:49.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:03:49.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.36.20:36785
08:03:49.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:03:49.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:03:49.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster11388303748095761808/ts-0/data
08:03:49.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster11388303748095761808/ts-0/wal
08:03:49.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:03:49.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:03:49.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.36.1:0
08:03:49.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:03:49.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:03:49.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:03:49.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster11388303748095761808/ts-0/data/info.pb
08:03:49.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.36.1
08:03:49.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:03:49.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.36.62:46005,127.23.36.61:41985,127.23.36.60:37459
08:03:49.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:03:49.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:03:49.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:03:49.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:03:49.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:03:49.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.36.1
08:03:49.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster11388303748095761808/ts-0/logs
08:03:49.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:03:49.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:03:49.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:03:49.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
08:03:49.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:03:49.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 8537ed5bb5e0e6ac1e481ff961e6b9d08cce7671
08:03:49.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:03:49.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 30 Apr 2026 07:43:22 UTC on e7f111948823
08:03:49.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11675
08:03:49.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:03:49.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:49.630976 23977 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:03:49.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:49.633391 23977 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:03:49.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:03:49.647701 24004 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:49.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:49.845394 23963 sys_catalog.cc:455] T 00000000000000000000000000000000 P d2bf70afa9c24023b22b92050d87c195 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "d2bf70afa9c24023b22b92050d87c195" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "91cc0cbb90c5488980c6a0da24388997" member_type: VOTER last_known_addr { host: "127.23.36.62" port: 46005 } } peers { permanent_uuid: "b53785e731cb44fd835d70e39bbe7a9c" member_type: VOTER last_known_addr { host: "127.23.36.61" port: 41985 } } peers { permanent_uuid: "d2bf70afa9c24023b22b92050d87c195" member_type: VOTER last_known_addr { host: "127.23.36.60" port: 37459 } } }
08:03:49.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:49.847601 23963 sys_catalog.cc:458] T 00000000000000000000000000000000 P d2bf70afa9c24023b22b92050d87c195 [sys.catalog]: This master's current role is: LEADER
08:03:49.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:49.876962 23970 catalog_manager.cc:1269] Loaded cluster ID: ad2e883dd84f401982a0078b10967bc7
08:03:49.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:49.878003 23970 catalog_manager.cc:1562] T 00000000000000000000000000000000 P b53785e731cb44fd835d70e39bbe7a9c: loading cluster ID for follower catalog manager: success
08:03:49.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:49.889909 23970 catalog_manager.cc:1584] T 00000000000000000000000000000000 P b53785e731cb44fd835d70e39bbe7a9c: acquiring CA information for follower catalog manager: success
08:03:49.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:49.899010 23970 catalog_manager.cc:1612] T 00000000000000000000000000000000 P b53785e731cb44fd835d70e39bbe7a9c: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
08:03:50.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:50.458624 23998 catalog_manager.cc:1269] Loaded cluster ID: ad2e883dd84f401982a0078b10967bc7
08:03:50.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:50.459360 23998 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 91cc0cbb90c5488980c6a0da24388997: loading cluster ID for follower catalog manager: success
08:03:50.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:50.508610 23998 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 91cc0cbb90c5488980c6a0da24388997: acquiring CA information for follower catalog manager: success
08:03:50.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:50.519678 23998 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 91cc0cbb90c5488980c6a0da24388997: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
08:03:51.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:03:49.648938 24005 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:51.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:03:51.190042 23977 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 1.544s user 0.403s sys 0.729s
08:03:51.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:03:51.047500 24003 debug-util.cc:398] Leaking SignalData structure 0x7b0800034ca0 after lost signal to thread 23977
08:03:51.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:03:51.192111 23977 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 1.546s user 0.403s sys 0.729s
08:03:51.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:51.198321 23977 server_base.cc:1061] running on GCE node
08:03:51.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:03:51.198550 24013 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:51.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:51.200487 23977 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:03:51.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:51.209031 23977 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:03:51.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:51.210857 23977 hybrid_clock.cc:648] HybridClock initialized: now 1777536231210778 us; error 86 us; skew 500 ppm
08:03:51.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:51.212260 23977 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:03:51.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:51.232134 23977 webserver.cc:492] Webserver started at http://127.23.36.1:40113/ using document root <none> and password file <none>
08:03:51.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:51.233978 23977 fs_manager.cc:362] Metadata directory not provided
08:03:51.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:51.234627 23977 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:03:51.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:51.235472 23977 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
08:03:51.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:51.242647 23977 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster11388303748095761808/ts-0/data/instance:
08:03:51.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "22c93ff50f2c43e483dab82cffd15517"
08:03:51.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-04-30 08:03:51 on dist-test-slave-h4q8"
08:03:51.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:51.244941 23977 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster11388303748095761808/ts-0/wal/instance:
08:03:51.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "22c93ff50f2c43e483dab82cffd15517"
08:03:51.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-04-30 08:03:51 on dist-test-slave-h4q8"
08:03:51.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:51.262217 23977 fs_manager.cc:696] Time spent creating directory manager: real 0.016s user 0.014s sys 0.000s
08:03:51.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:51.276322 24018 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:03:51.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:51.278410 23977 fs_manager.cc:730] Time spent opening block manager: real 0.011s user 0.004s sys 0.006s
08:03:51.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:51.279301 23977 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster11388303748095761808/ts-0/data,/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster11388303748095761808/ts-0/wal
08:03:51.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "22c93ff50f2c43e483dab82cffd15517"
08:03:51.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-04-30 08:03:51 on dist-test-slave-h4q8"
08:03:51.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:51.280225 23977 fs_report.cc:389] FS layout report
08:03:51.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:03:51.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster11388303748095761808/ts-0/wal
08:03:51.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster11388303748095761808/ts-0/wal
08:03:51.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster11388303748095761808/ts-0/data/data
08:03:51.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:03:51.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:03:51.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:03:51.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:03:51.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:03:51.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:03:51.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:03:51.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:03:51.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:03:51.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:03:51.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:03:51.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:51.378561 23977 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:03:51.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:51.381176 23977 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:03:51.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:51.393144 23977 kserver.cc:163] Server-wide thread pool size limit: 3276
08:03:51.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:51.410589 23977 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
08:03:51.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:51.424424 23977 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
08:03:51.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:51.425205 23977 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.001s user 0.000s sys 0.001s
08:03:51.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:51.425810 23977 ts_tablet_manager.cc:616] Registered 0 tablets
08:03:51.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:51.426231 23977 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
08:03:52.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:52.340095 23977 rpc_server.cc:307] RPC server started. Bound to: 127.23.36.1:42993
08:03:52.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:52.340148 24130 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.23.36.1:42993 every 8 connection(s)
08:03:52.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:52.343456 23977 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster11388303748095761808/ts-0/data/info.pb
08:03:52.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:52.353348 23696 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskf7pa4T/build/tsan/bin/kudu as pid 23977
08:03:52.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:52.355270 23696 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster11388303748095761808/ts-0/wal/instance
08:03:52.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:52.377293 23696 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskf7pa4T/build/tsan/bin/kudu
08:03:52.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskf7pa4T/build/tsan/bin/kudu
08:03:52.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster11388303748095761808/ts-1/wal
08:03:52.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster11388303748095761808/ts-1/data
08:03:52.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:03:52.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:03:52.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:03:52.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:03:52.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:03:52.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:03:52.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster11388303748095761808/ts-1/logs
08:03:52.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster11388303748095761808/ts-1/data/info.pb
08:03:52.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:03:52.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:03:52.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:03:52.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:03:52.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:03:52.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:03:52.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:03:52.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:03:52.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
08:03:52.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:03:52.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.36.2:0
08:03:52.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.36.2
08:03:52.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.36.2
08:03:52.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:03:52.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.36.62:46005,127.23.36.61:41985,127.23.36.60:37459
08:03:52.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.36.20:36785
08:03:52.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:03:52.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:03:52.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
08:03:52.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:52.438333 24131 heartbeater.cc:344] Connected to a master server at 127.23.36.60:37459
08:03:52.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:52.438848 24131 heartbeater.cc:461] Registering TS with master...
08:03:52.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:52.440229 24131 heartbeater.cc:507] Master 127.23.36.60:37459 requested a full tablet report, sending...
08:03:52.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:52.441581 24132 heartbeater.cc:344] Connected to a master server at 127.23.36.61:41985
08:03:52.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:52.442009 24132 heartbeater.cc:461] Registering TS with master...
08:03:52.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:52.442963 24132 heartbeater.cc:507] Master 127.23.36.61:41985 requested a full tablet report, sending...
08:03:52.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:52.447362 23816 ts_manager.cc:194] Registered new tserver with Master: 22c93ff50f2c43e483dab82cffd15517 (127.23.36.1:42993)
08:03:52.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:52.451043 23897 ts_manager.cc:194] Registered new tserver with Master: 22c93ff50f2c43e483dab82cffd15517 (127.23.36.1:42993)
08:03:52.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:52.456943 23897 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.23.36.1:35373
08:03:52.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:52.472302 24135 heartbeater.cc:344] Connected to a master server at 127.23.36.62:46005
08:03:52.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:52.472702 24135 heartbeater.cc:461] Registering TS with master...
08:03:52.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:52.473383 24135 heartbeater.cc:507] Master 127.23.36.62:46005 requested a full tablet report, sending...
08:03:52.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:52.476543 23741 ts_manager.cc:194] Registered new tserver with Master: 22c93ff50f2c43e483dab82cffd15517 (127.23.36.1:42993)
08:03:52.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:03:52.935106 24141 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
08:03:52.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:03:52.935691 24141 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:03:52.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:03:52.936259 24141 flags.cc:432] Enabled unsafe flag: --never_fsync=true
08:03:52.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:03:52.971591 24141 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
08:03:52.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:03:52.972559 24141 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.23.36.2
08:03:53.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:53.037449 24141 tablet_server_runner.cc:78] Tablet server non-default flags:
08:03:53.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:03:53.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.36.20:36785
08:03:53.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:03:53.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:03:53.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster11388303748095761808/ts-1/data
08:03:53.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster11388303748095761808/ts-1/wal
08:03:53.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:03:53.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:03:53.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.36.2:0
08:03:53.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:03:53.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:03:53.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:03:53.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster11388303748095761808/ts-1/data/info.pb
08:03:53.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.36.2
08:03:53.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:03:53.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.36.62:46005,127.23.36.61:41985,127.23.36.60:37459
08:03:53.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:03:53.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:03:53.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:03:53.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:03:53.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:03:53.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.36.2
08:03:53.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster11388303748095761808/ts-1/logs
08:03:53.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:03:53.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:03:53.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:03:53.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
08:03:53.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:03:53.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 8537ed5bb5e0e6ac1e481ff961e6b9d08cce7671
08:03:53.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:03:53.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 30 Apr 2026 07:43:22 UTC on e7f111948823
08:03:53.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11675
08:03:53.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:03:53.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:53.039211 24141 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:03:53.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:53.041392 24141 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:03:53.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:03:53.057277 24149 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:53.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:53.462056 24131 heartbeater.cc:499] Master 127.23.36.60:37459 was elected leader, sending a full tablet report...
08:03:55.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:03:54.463240 24148 debug-util.cc:398] Leaking SignalData structure 0x7b0800034ca0 after lost signal to thread 24141
08:03:55.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:03:55.101785 24148 kernel_stack_watchdog.cc:198] Thread 24141 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 403ms:
08:03:55.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
08:03:55.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
08:03:55.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
08:03:55.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
08:03:55.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:03:53.059196 24150 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:55.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:03:55.103118 24141 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 2.045s user 0.719s sys 1.151s
08:03:55.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:03:55.103521 24141 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 2.046s user 0.719s sys 1.151s
08:03:55.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:03:55.106792 24151 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2043 milliseconds
08:03:55.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:03:55.108501 24152 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:55.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:55.108561 24141 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
08:03:55.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:55.110098 24141 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:03:55.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:55.112255 24141 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:03:55.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:55.113584 24141 hybrid_clock.cc:648] HybridClock initialized: now 1777536235113552 us; error 64 us; skew 500 ppm
08:03:55.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:55.114485 24141 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:03:55.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:55.120409 24141 webserver.cc:492] Webserver started at http://127.23.36.2:41231/ using document root <none> and password file <none>
08:03:55.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:55.121268 24141 fs_manager.cc:362] Metadata directory not provided
08:03:55.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:55.121450 24141 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:03:55.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:55.121899 24141 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
08:03:55.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:55.126279 24141 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster11388303748095761808/ts-1/data/instance:
08:03:55.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "14b1a93219ac4891aec826358baa97d2"
08:03:55.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-04-30 08:03:55 on dist-test-slave-h4q8"
08:03:55.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:55.127319 24141 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster11388303748095761808/ts-1/wal/instance:
08:03:55.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "14b1a93219ac4891aec826358baa97d2"
08:03:55.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-04-30 08:03:55 on dist-test-slave-h4q8"
08:03:55.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:55.134474 24141 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.006s sys 0.002s
08:03:55.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:55.140981 24160 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:03:55.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:55.142221 24141 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.005s sys 0.000s
08:03:55.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:55.142580 24141 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster11388303748095761808/ts-1/data,/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster11388303748095761808/ts-1/wal
08:03:55.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "14b1a93219ac4891aec826358baa97d2"
08:03:55.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-04-30 08:03:55 on dist-test-slave-h4q8"
08:03:55.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:55.142894 24141 fs_report.cc:389] FS layout report
08:03:55.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:03:55.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster11388303748095761808/ts-1/wal
08:03:55.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster11388303748095761808/ts-1/wal
08:03:55.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster11388303748095761808/ts-1/data/data
08:03:55.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:03:55.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:03:55.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:03:55.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:03:55.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:03:55.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:03:55.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:03:55.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:03:55.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:03:55.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:03:55.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:03:55.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:55.200933 24141 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:03:55.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:55.202381 24141 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:03:55.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:55.202770 24141 kserver.cc:163] Server-wide thread pool size limit: 3276
08:03:55.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:55.205274 24141 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
08:03:55.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:55.209293 24141 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
08:03:55.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:55.209529 24141 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
08:03:55.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:55.209848 24141 ts_tablet_manager.cc:616] Registered 0 tablets
08:03:55.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:55.210096 24141 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
08:03:55.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:55.367483 24141 rpc_server.cc:307] RPC server started. Bound to: 127.23.36.2:45713
08:03:55.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:55.367537 24272 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.23.36.2:45713 every 8 connection(s)
08:03:55.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:55.370235 24141 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster11388303748095761808/ts-1/data/info.pb
08:03:55.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:55.378091 23696 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskf7pa4T/build/tsan/bin/kudu as pid 24141
08:03:55.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:55.379143 23696 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster11388303748095761808/ts-1/wal/instance
08:03:55.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:55.398398 23696 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskf7pa4T/build/tsan/bin/kudu
08:03:55.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskf7pa4T/build/tsan/bin/kudu
08:03:55.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster11388303748095761808/ts-2/wal
08:03:55.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster11388303748095761808/ts-2/data
08:03:55.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:03:55.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:03:55.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:03:55.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:03:55.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:03:55.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:03:55.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster11388303748095761808/ts-2/logs
08:03:55.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster11388303748095761808/ts-2/data/info.pb
08:03:55.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:03:55.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:03:55.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:03:55.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:03:55.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:03:55.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:03:55.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:03:55.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:03:55.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
08:03:55.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:03:55.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.36.3:0
08:03:55.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.36.3
08:03:55.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.36.3
08:03:55.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:03:55.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.36.62:46005,127.23.36.61:41985,127.23.36.60:37459
08:03:55.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.36.20:36785
08:03:55.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:03:55.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:03:55.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
08:03:55.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:55.413064 24274 heartbeater.cc:344] Connected to a master server at 127.23.36.61:41985
08:03:55.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:55.413599 24274 heartbeater.cc:461] Registering TS with master...
08:03:55.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:55.415006 24274 heartbeater.cc:507] Master 127.23.36.61:41985 requested a full tablet report, sending...
08:03:55.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:55.418545 23816 ts_manager.cc:194] Registered new tserver with Master: 14b1a93219ac4891aec826358baa97d2 (127.23.36.2:45713)
08:03:55.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:55.421206 24273 heartbeater.cc:344] Connected to a master server at 127.23.36.60:37459
08:03:55.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:55.421466 24273 heartbeater.cc:461] Registering TS with master...
08:03:55.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:55.422303 24273 heartbeater.cc:507] Master 127.23.36.60:37459 requested a full tablet report, sending...
08:03:55.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:55.425303 23897 ts_manager.cc:194] Registered new tserver with Master: 14b1a93219ac4891aec826358baa97d2 (127.23.36.2:45713)
08:03:55.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:55.426614 24275 heartbeater.cc:344] Connected to a master server at 127.23.36.62:46005
08:03:55.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:55.426963 24275 heartbeater.cc:461] Registering TS with master...
08:03:55.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:55.426977 23897 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.23.36.2:56493
08:03:55.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:55.427834 24275 heartbeater.cc:507] Master 127.23.36.62:46005 requested a full tablet report, sending...
08:03:55.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:55.431411 23741 ts_manager.cc:194] Registered new tserver with Master: 14b1a93219ac4891aec826358baa97d2 (127.23.36.2:45713)
08:03:55.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:03:55.736657 24283 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
08:03:55.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:03:55.737130 24283 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:03:55.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:03:55.737690 24283 flags.cc:432] Enabled unsafe flag: --never_fsync=true
08:03:55.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:03:55.770584 24283 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
08:03:55.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:03:55.771337 24283 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.23.36.3
08:03:55.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:55.808300 24283 tablet_server_runner.cc:78] Tablet server non-default flags:
08:03:55.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:03:55.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.36.20:36785
08:03:55.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:03:55.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:03:55.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster11388303748095761808/ts-2/data
08:03:55.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster11388303748095761808/ts-2/wal
08:03:55.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:03:55.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:03:55.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.36.3:0
08:03:55.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:03:55.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:03:55.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:03:55.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster11388303748095761808/ts-2/data/info.pb
08:03:55.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.36.3
08:03:55.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:03:55.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.36.62:46005,127.23.36.61:41985,127.23.36.60:37459
08:03:55.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:03:55.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:03:55.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:03:55.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:03:55.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:03:55.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.36.3
08:03:55.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster11388303748095761808/ts-2/logs
08:03:55.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:03:55.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:03:55.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:03:55.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
08:03:55.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:03:55.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 8537ed5bb5e0e6ac1e481ff961e6b9d08cce7671
08:03:55.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:03:55.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 30 Apr 2026 07:43:22 UTC on e7f111948823
08:03:55.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11675
08:03:55.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:03:55.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:55.809804 24283 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:03:55.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:55.811527 24283 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:03:55.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:03:55.824837 24289 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:56.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:03:56.021688 24127 debug-util.cc:398] Leaking SignalData structure 0x7b08000ac1a0 after lost signal to thread 23999
08:03:56.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:03:56.023577 24127 debug-util.cc:398] Leaking SignalData structure 0x7b08000acc40 after lost signal to thread 24130
08:03:56.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:56.433283 24273 heartbeater.cc:499] Master 127.23.36.60:37459 was elected leader, sending a full tablet report...
08:03:57.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:03:55.826077 24290 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:57.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:03:57.228533 24288 debug-util.cc:398] Leaking SignalData structure 0x7b0800034ca0 after lost signal to thread 24283
08:03:57.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:03:57.604508 24288 kernel_stack_watchdog.cc:198] Thread 24283 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 400ms:
08:03:57.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
08:03:57.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
08:03:57.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
08:03:57.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
08:03:57.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:03:57.606220 24291 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection timed out after 1779 milliseconds
08:03:57.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:03:57.607304 24283 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.781s user 0.000s sys 0.002s
08:03:57.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:03:57.607664 24283 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.782s user 0.000s sys 0.002s
08:03:57.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:03:57.608281 24292 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:57.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:57.608333 24283 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
08:03:57.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:57.613858 24283 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:03:57.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:57.616600 24283 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:03:57.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:57.618263 24283 hybrid_clock.cc:648] HybridClock initialized: now 1777536237618221 us; error 46 us; skew 500 ppm
08:03:57.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:57.619390 24283 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:03:57.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:57.627524 24283 webserver.cc:492] Webserver started at http://127.23.36.3:36367/ using document root <none> and password file <none>
08:03:57.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:57.629123 24283 fs_manager.cc:362] Metadata directory not provided
08:03:57.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:57.629416 24283 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:03:57.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:57.630019 24283 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
08:03:57.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:57.637336 24283 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster11388303748095761808/ts-2/data/instance:
08:03:57.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6bc88bbf15194d8d9a49ea5683c8198f"
08:03:57.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-04-30 08:03:57 on dist-test-slave-h4q8"
08:03:57.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:57.639005 24283 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster11388303748095761808/ts-2/wal/instance:
08:03:57.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6bc88bbf15194d8d9a49ea5683c8198f"
08:03:57.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-04-30 08:03:57 on dist-test-slave-h4q8"
08:03:57.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:57.649151 24283 fs_manager.cc:696] Time spent creating directory manager: real 0.009s user 0.010s sys 0.000s
08:03:57.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:57.656828 24300 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:03:57.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:57.658123 24283 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.001s
08:03:57.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:57.658515 24283 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster11388303748095761808/ts-2/data,/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster11388303748095761808/ts-2/wal
08:03:57.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6bc88bbf15194d8d9a49ea5683c8198f"
08:03:57.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-04-30 08:03:57 on dist-test-slave-h4q8"
08:03:57.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:57.658917 24283 fs_report.cc:389] FS layout report
08:03:57.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:03:57.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster11388303748095761808/ts-2/wal
08:03:57.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster11388303748095761808/ts-2/wal
08:03:57.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster11388303748095761808/ts-2/data/data
08:03:57.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:03:57.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:03:57.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:03:57.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:03:57.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:03:57.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:03:57.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:03:57.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:03:57.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:03:57.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:03:57.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:03:57.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:57.714367 24283 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:03:57.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:57.715996 24283 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:03:57.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:57.716562 24283 kserver.cc:163] Server-wide thread pool size limit: 3276
08:03:57.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:57.719986 24283 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
08:03:57.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:57.725428 24283 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
08:03:57.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:57.725677 24283 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
08:03:57.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:57.726079 24283 ts_tablet_manager.cc:616] Registered 0 tablets
08:03:57.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:57.726292 24283 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
08:03:57.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:57.925787 24283 rpc_server.cc:307] RPC server started. Bound to: 127.23.36.3:44163
08:03:57.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:57.925940 24412 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.23.36.3:44163 every 8 connection(s)
08:03:57.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:57.928485 24283 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster11388303748095761808/ts-2/data/info.pb
08:03:57.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:57.929972 23696 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskf7pa4T/build/tsan/bin/kudu as pid 24283
08:03:57.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:57.930490 23696 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster11388303748095761808/ts-2/wal/instance
08:03:58.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:58.002372 24413 heartbeater.cc:344] Connected to a master server at 127.23.36.60:37459
08:03:58.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:58.002919 24413 heartbeater.cc:461] Registering TS with master...
08:03:58.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:58.004943 24413 heartbeater.cc:507] Master 127.23.36.60:37459 requested a full tablet report, sending...
08:03:58.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:58.008198 23897 ts_manager.cc:194] Registered new tserver with Master: 6bc88bbf15194d8d9a49ea5683c8198f (127.23.36.3:44163)
08:03:58.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:58.009074 24414 heartbeater.cc:344] Connected to a master server at 127.23.36.61:41985
08:03:58.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:58.009465 24414 heartbeater.cc:461] Registering TS with master...
08:03:58.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:58.010481 24414 heartbeater.cc:507] Master 127.23.36.61:41985 requested a full tablet report, sending...
08:03:58.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:58.010658 23897 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.23.36.3:50209
08:03:58.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:58.013259 23816 ts_manager.cc:194] Registered new tserver with Master: 6bc88bbf15194d8d9a49ea5683c8198f (127.23.36.3:44163)
08:03:58.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:58.013790 24419 heartbeater.cc:344] Connected to a master server at 127.23.36.62:46005
08:03:58.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:58.014127 24419 heartbeater.cc:461] Registering TS with master...
08:03:58.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:58.015159 24419 heartbeater.cc:507] Master 127.23.36.62:46005 requested a full tablet report, sending...
08:03:58.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:58.017592 23741 ts_manager.cc:194] Registered new tserver with Master: 6bc88bbf15194d8d9a49ea5683c8198f (127.23.36.3:44163)
08:03:58.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:58.031894 23696 external_mini_cluster.cc:949] 3 TS(s) registered with all masters
08:03:58.044 [DEBUG - main] (MiniKuduCluster.java:195) Response:
08:03:58.046 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
08:03:58.059 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.23.36.62"
port: 46005
}
bound_http_address {
host: "127.23.36.62"
port: 41239
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.23.36.61"
port: 41985
}
bound_http_address {
host: "127.23.36.61"
port: 39255
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.23.36.60"
port: 37459
}
bound_http_address {
host: "127.23.36.60"
port: 41955
}
}
}
08:03:58.063 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
08:03:58.066 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.23.36.1"
port: 42993
}
bound_http_address {
host: "127.23.36.1"
port: 40113
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.23.36.2"
port: 45713
}
bound_http_address {
host: "127.23.36.2"
port: 41231
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.23.36.3"
port: 44163
}
bound_http_address {
host: "127.23.36.3"
port: 36367
}
}
}
08:03:58.066 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
08:03:58.286 [DEBUG - main] (CompositeStrategy.java:59) Unable to load modern Subject methods, relying only on legacy methods
java.lang.NoSuchMethodException: javax.security.auth.Subject.current()
at java.base/java.lang.Class.getDeclaredMethod(Class.java:2677)
at org.apache.kudu.client.internals.ModernStrategy.<init>(ModernStrategy.java:51)
at org.apache.kudu.client.internals.CompositeStrategy.<init>(CompositeStrategy.java:52)
at org.apache.kudu.client.internals.CompositeStrategy.<clinit>(CompositeStrategy.java:41)
at org.apache.kudu.client.internals.SecurityManagerCompatibility.get(SecurityManagerCompatibility.java:43)
at org.apache.kudu.client.SecurityContext.setupSubject(SecurityContext.java:155)
at org.apache.kudu.client.SecurityContext.<init>(SecurityContext.java:138)
at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:431)
at org.apache.kudu.client.AsyncKuduClient$AsyncKuduClientBuilder.build(AsyncKuduClient.java:3109)
at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:161)
at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runners.Suite.runChild(Suite.java:128)
at org.junit.runners.Suite.runChild(Suite.java:27)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runner.JUnitCore.run(JUnitCore.java:137)
at org.junit.runner.JUnitCore.run(JUnitCore.java:115)
at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77)
at org.junit.runner.JUnitCore.main(JUnitCore.java:36)
java.lang.NoSuchMethodException: javax.security.auth.Subject.current()
at java.base/java.lang.Class.getDeclaredMethod(Class.java:2677)
at org.apache.kudu.client.internals.ModernStrategy.<init>(ModernStrategy.java:51)
at org.apache.kudu.client.internals.CompositeStrategy.<init>(CompositeStrategy.java:52)
at org.apache.kudu.client.internals.CompositeStrategy.<clinit>(CompositeStrategy.java:41)
at org.apache.kudu.client.internals.SecurityManagerCompatibility.get(SecurityManagerCompatibility.java:43)
at org.apache.kudu.client.SecurityContext.setupSubject(SecurityContext.java:155)
at org.apache.kudu.client.SecurityContext.<init>(SecurityContext.java:138)
at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:431)
at org.apache.kudu.client.AsyncKuduClient$AsyncKuduClientBuilder.build(AsyncKuduClient.java:3109)
at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:161)
at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runners.Suite.runChild(Suite.java:128)
at org.junit.runners.Suite.runChild(Suite.java:27)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runner.JUnitCore.run(JUnitCore.java:137)
at org.junit.runner.JUnitCore.run(JUnitCore.java:115)
at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77)
at org.junit.runner.JUnitCore.main(JUnitCore.java:36)
08:03:58.360 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
08:03:58.566 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskf7pa4T/test-tmp
08:03:58.567 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
08:03:58.567 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskf7pa4T/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
08:03:58.592 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772"
extra_master_flags: "--webserver_require_spnego=false"
extra_master_flags: "--authn_token_validity_seconds=10"
extra_tserver_flags: "--webserver_require_spnego=false"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
08:03:58.996 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:03:58.997 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
08:03:59.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:59.000170 24425 mini_oidc.cc:150] Starting JWKS server
08:03:59.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:59.003880 24425 webserver.cc:492] Webserver started at http://127.0.0.1:43537/ using document root <none> and password file <none>
08:03:59.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:59.004395 24425 mini_oidc.cc:186] Starting OIDC Discovery server
08:03:59.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:59.006017 24425 webserver.cc:492] Webserver started at http://127.0.0.1:44981/ using document root <none> and password file <none>
08:03:59.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:03:59.017052 24413 heartbeater.cc:499] Master 127.23.36.60:37459 was elected leader, sending a full tablet report...
08:03:59.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
08:03:59.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/krb5kdc/principal' for realm 'KRBTEST.COM',
08:03:59.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
08:03:59.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 30 08:03:59 dist-test-slave-h4q8 krb5kdc[24437](info): setting up network...
08:03:59.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
08:03:59.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 30 08:03:59 dist-test-slave-h4q8 krb5kdc[24437](info): set up 2 sockets
08:03:59.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 30 08:03:59 dist-test-slave-h4q8 krb5kdc[24437](info): commencing operation
08:03:59.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
08:04:00.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:00.557001 24269 debug-util.cc:398] Leaking SignalData structure 0x7b08000ad660 after lost signal to thread 24143
08:04:00.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:00.558100 24269 debug-util.cc:398] Leaking SignalData structure 0x7b08000adce0 after lost signal to thread 24272
08:04:01.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:01.174556 24425 mini_kdc.cc:121] Time spent starting KDC: real 2.131s user 0.000s sys 0.021s
08:04:01.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
08:04:01.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
08:04:01.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
08:04:01.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
08:04:01.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
08:04:01.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
08:04:01.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
08:04:01.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
08:04:01.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
08:04:01.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
08:04:01.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/krb5kdc/test-user.keytab.
08:04:01.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/krb5kdc/test-user.keytab.
08:04:01.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 30 08:04:01 dist-test-slave-h4q8 krb5kdc[24437](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777536241, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:04:01.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
08:04:01.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-04-30T08:04:01Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
08:04:01.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-04-30T08:04:01Z Disabled control of system clock
08:04:01.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.23.218.126@KRBTEST.COM; defaulting to no policy
08:04:01.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:04:01.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.23.218.126@KRBTEST.COM" created.
08:04:01.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:04:01.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.23.218.126 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/krb5kdc/kudu.keytab.
08:04:01.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.23.218.126 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/krb5kdc/kudu.keytab.
08:04:01.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.23.218.126@KRBTEST.COM; defaulting to no policy
08:04:01.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:04:01.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.23.218.126@KRBTEST.COM" created.
08:04:01.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:04:01.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.23.218.126 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/krb5kdc/kudu.keytab.
08:04:01.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.23.218.126 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/krb5kdc/kudu.keytab.
08:04:01.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:01.415159 24425 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskf7pa4T/build/tsan/bin/kudu
08:04:01.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskf7pa4T/build/tsan/bin/kudu
08:04:01.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/master-0/wal
08:04:01.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/master-0/data
08:04:01.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:04:01.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:04:01.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:04:01.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:04:01.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:04:01.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:04:01.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/master-0/logs
08:04:01.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/master-0/data/info.pb
08:04:01.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:04:01.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:04:01.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:04:01.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:04:01.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:04:01.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:04:01.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:04:01.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:04:01.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:04:01.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:04:01.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:04:01.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:04:01.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.218.126:42861
08:04:01.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.218.126
08:04:01.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:04:01.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/krb5kdc/kudu.keytab
08:04:01.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.23.218.126
08:04:01.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:04:01.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
08:04:01.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
08:04:01.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
08:04:01.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.218.84:37547
08:04:01.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:04:01.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:04:01.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:04:01.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:04:01.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.218.126:42861,127.23.218.125:44351,127.23.218.124:32793
08:04:01.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:04:01.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43537/jwks.json
08:04:01.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
08:04:01.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
08:04:01.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:01.828632 24454 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
08:04:01.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:01.829186 24454 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:04:01.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:01.829718 24454 flags.cc:432] Enabled unsafe flag: --never_fsync=true
08:04:01.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:01.863682 24454 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
08:04:01.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:01.863916 24454 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
08:04:01.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:01.864192 24454 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
08:04:01.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:01.864379 24454 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
08:04:01.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:01.864681 24454 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
08:04:01.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:01.864894 24454 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
08:04:01.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:01.865120 24454 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:43537/jwks.json
08:04:01.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:01.905553 24454 master_runner.cc:386] Master server non-default flags:
08:04:01.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:04:01.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.218.84:37547
08:04:01.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:04:01.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:04:01.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/master-0/data
08:04:01.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/master-0/wal
08:04:01.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
08:04:01.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:04:01.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.218.126:42861,127.23.218.125:44351,127.23.218.124:32793
08:04:01.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/krb5kdc/kudu.keytab
08:04:01.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.23.218.126
08:04:01.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:04:01.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:04:01.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:04:01.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.218.126:42861
08:04:01.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:04:01.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:04:01.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:04:01.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43537/jwks.json
08:04:01.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:04:01.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
08:04:01.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
08:04:01.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:04:01.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:04:01.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/master-0/data/info.pb
08:04:01.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.218.126
08:04:01.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:04:01.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:04:01.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:04:01.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:04:01.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:04:01.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:04:01.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/master-0/logs
08:04:01.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:04:01.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:04:01.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:04:01.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:04:01.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:04:01.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 8537ed5bb5e0e6ac1e481ff961e6b9d08cce7671
08:04:01.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:04:01.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 30 Apr 2026 07:43:22 UTC on e7f111948823
08:04:01.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11675
08:04:01.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:04:01.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:01.906908 24454 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:04:01.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:01.907848 24454 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:04:01.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:01.922853 24460 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:04:03.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:03.323562 24459 debug-util.cc:398] Leaking SignalData structure 0x7b0800028d00 after lost signal to thread 24454
08:04:03.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:03.648839 24459 kernel_stack_watchdog.cc:198] Thread 24454 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 396ms:
08:04:03.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
08:04:03.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
08:04:03.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
08:04:03.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
08:04:03.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:03.649183 24454 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.724s user 0.615s sys 0.959s
08:04:03.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:03.649612 24454 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.725s user 0.615s sys 0.959s
08:04:03.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:01.928398 24461 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:04:03.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:03.650641 24462 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection timed out after 1725 milliseconds
08:04:03.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:03.651518 24454 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
08:04:03.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:03.651531 24463 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:04:03.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:03.654753 24454 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:04:03.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:03.656976 24454 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:04:03.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:03.658314 24454 hybrid_clock.cc:648] HybridClock initialized: now 1777536243658272 us; error 64 us; skew 500 ppm
08:04:03.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:03.659053 24454 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:04:03.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 30 08:04:03 dist-test-slave-h4q8 krb5kdc[24437](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777536243, etypes {rep=17 tkt=17 ses=17}, kudu/127.23.218.126@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:04:03.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:03.664682 24454 init.cc:377] Logged in from keytab as kudu/127.23.218.126@KRBTEST.COM (short username kudu)
08:04:03.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:03.669355 24454 webserver.cc:492] Webserver started at http://127.23.218.126:45411/ using document root <none> and password file <none>
08:04:03.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:03.670265 24454 fs_manager.cc:362] Metadata directory not provided
08:04:03.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:03.670446 24454 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:04:03.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:03.670853 24454 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
08:04:03.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:03.674916 24454 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/master-0/data/instance:
08:04:03.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "85f6c126e6444b7bae846a0c871b0e18"
08:04:03.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-04-30 08:04:03 on dist-test-slave-h4q8"
08:04:03.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:03.675912 24454 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/master-0/wal/instance:
08:04:03.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "85f6c126e6444b7bae846a0c871b0e18"
08:04:03.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-04-30 08:04:03 on dist-test-slave-h4q8"
08:04:03.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:03.683131 24454 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.008s sys 0.000s
08:04:03.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:03.689011 24475 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:04:03.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:03.690851 24454 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.003s sys 0.000s
08:04:03.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:03.691272 24454 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/master-0/data,/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/master-0/wal
08:04:03.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "85f6c126e6444b7bae846a0c871b0e18"
08:04:03.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-04-30 08:04:03 on dist-test-slave-h4q8"
08:04:03.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:03.691686 24454 fs_report.cc:389] FS layout report
08:04:03.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:04:03.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/master-0/wal
08:04:03.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/master-0/wal
08:04:03.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/master-0/data/data
08:04:03.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:04:03.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:04:03.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:04:03.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:04:03.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:04:03.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:04:03.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:04:03.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:04:03.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:04:03.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:04:03.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:04:03.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:03.762228 24454 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:04:03.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:03.770597 24454 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:04:03.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:03.771019 24454 kserver.cc:163] Server-wide thread pool size limit: 3276
08:04:03.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:03.854323 24454 rpc_server.cc:307] RPC server started. Bound to: 127.23.218.126:42861
08:04:03.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:03.854391 24526 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.23.218.126:42861 every 8 connection(s)
08:04:03.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:03.856897 24454 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/master-0/data/info.pb
08:04:03.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:03.863305 24527 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
08:04:03.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:03.864809 24425 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskf7pa4T/build/tsan/bin/kudu as pid 24454
08:04:03.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:03.865270 24425 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/master-0/wal/instance
08:04:03.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:03.874935 24527 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.218.126" port: 42861 } has no permanent_uuid. Determining permanent_uuid...
08:04:03.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.23.218.125@KRBTEST.COM; defaulting to no policy
08:04:03.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:04:03.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.23.218.125@KRBTEST.COM" created.
08:04:03.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 30 08:04:03 dist-test-slave-h4q8 krb5kdc[24437](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777536243, etypes {rep=17 tkt=17 ses=17}, kudu/127.23.218.126@KRBTEST.COM for kudu/127.23.218.126@KRBTEST.COM
08:04:03.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:04:03.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.23.218.125 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/krb5kdc/kudu.keytab.
08:04:03.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.23.218.125 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/krb5kdc/kudu.keytab.
08:04:03.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:03.917007 24527 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.218.125" port: 44351 } has no permanent_uuid. Determining permanent_uuid...
08:04:03.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:03.919512 24479 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.23.218.125:44351: connect: Connection refused (error 111)
08:04:03.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:03.923627 24527 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.218.125:44351: Network error: Client connection negotiation failed: client connection to 127.23.218.125:44351: connect: Connection refused (error 111)
08:04:03.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.23.218.125@KRBTEST.COM; defaulting to no policy
08:04:03.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:04:03.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.23.218.125@KRBTEST.COM" created.
08:04:03.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:04:03.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.23.218.125 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/krb5kdc/kudu.keytab.
08:04:03.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.23.218.125 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/krb5kdc/kudu.keytab.
08:04:03.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:03.950678 24425 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskf7pa4T/build/tsan/bin/kudu
08:04:03.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskf7pa4T/build/tsan/bin/kudu
08:04:03.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/master-1/wal
08:04:03.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/master-1/data
08:04:03.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:04:03.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:04:03.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:04:03.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:04:03.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:04:03.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:04:03.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/master-1/logs
08:04:03.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/master-1/data/info.pb
08:04:03.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:04:03.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:04:03.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:04:03.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:04:03.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:04:03.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:04:03.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:04:03.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:04:03.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:04:03.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:04:03.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:04:03.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:04:03.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.218.125:44351
08:04:03.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.218.125
08:04:03.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:04:03.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/krb5kdc/kudu.keytab
08:04:03.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.23.218.125
08:04:03.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:04:03.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
08:04:03.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
08:04:03.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
08:04:03.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.218.84:37547
08:04:03.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:04:03.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:04:03.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:04:03.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:04:03.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.218.126:42861,127.23.218.125:44351,127.23.218.124:32793
08:04:03.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:04:03.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43537/jwks.json
08:04:03.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
08:04:03.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
08:04:03.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:03.973132 24527 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.218.125" port: 44351 } attempt: 1
08:04:03.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:03.977867 24527 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.218.125:44351: Network error: Client connection negotiation failed: client connection to 127.23.218.125:44351: connect: Connection refused (error 111)
08:04:04.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:04.046329 24527 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.218.125" port: 44351 } attempt: 2
08:04:04.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:04.051560 24527 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.218.125:44351: Network error: Client connection negotiation failed: client connection to 127.23.218.125:44351: connect: Connection refused (error 111)
08:04:04.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:04.143018 24527 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.218.125" port: 44351 } attempt: 3
08:04:04.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:04.147300 24527 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.218.125:44351: Network error: Client connection negotiation failed: client connection to 127.23.218.125:44351: connect: Connection refused (error 111)
08:04:04.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:04.278085 24536 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
08:04:04.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:04.278761 24536 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:04:04.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:04.279464 24536 flags.cc:432] Enabled unsafe flag: --never_fsync=true
08:04:04.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:04.290700 24527 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.218.125" port: 44351 } attempt: 4
08:04:04.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:04.295454 24527 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.218.125:44351: Network error: Client connection negotiation failed: client connection to 127.23.218.125:44351: connect: Connection refused (error 111)
08:04:04.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:04.315627 24536 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
08:04:04.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:04.315860 24536 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
08:04:04.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:04.316138 24536 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
08:04:04.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:04.316337 24536 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
08:04:04.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:04.316531 24536 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
08:04:04.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:04.316730 24536 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
08:04:04.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:04.316874 24536 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:43537/jwks.json
08:04:04.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:04.342458 24409 debug-util.cc:398] Leaking SignalData structure 0x7b08000ba0c0 after lost signal to thread 24284
08:04:04.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:04.343438 24409 debug-util.cc:398] Leaking SignalData structure 0x7b08000ba9c0 after lost signal to thread 24412
08:04:04.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:04.355584 24536 master_runner.cc:386] Master server non-default flags:
08:04:04.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:04:04.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.218.84:37547
08:04:04.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:04:04.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:04:04.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/master-1/data
08:04:04.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/master-1/wal
08:04:04.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
08:04:04.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:04:04.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.218.126:42861,127.23.218.125:44351,127.23.218.124:32793
08:04:04.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/krb5kdc/kudu.keytab
08:04:04.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.23.218.125
08:04:04.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:04:04.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:04:04.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:04:04.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.218.125:44351
08:04:04.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:04:04.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:04:04.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:04:04.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43537/jwks.json
08:04:04.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:04:04.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
08:04:04.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
08:04:04.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:04:04.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:04:04.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/master-1/data/info.pb
08:04:04.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.218.125
08:04:04.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:04:04.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:04:04.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:04:04.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:04:04.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:04:04.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:04:04.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/master-1/logs
08:04:04.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:04:04.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:04:04.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:04:04.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:04:04.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:04:04.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 8537ed5bb5e0e6ac1e481ff961e6b9d08cce7671
08:04:04.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:04:04.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 30 Apr 2026 07:43:22 UTC on e7f111948823
08:04:04.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11675
08:04:04.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:04:04.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:04.356853 24536 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:04:04.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:04.358071 24536 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:04:04.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:04.370659 24542 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:04:04.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:04.594856 24527 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.218.125" port: 44351 } attempt: 5
08:04:04.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:04.600688 24527 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.218.125:44351: Network error: Client connection negotiation failed: client connection to 127.23.218.125:44351: connect: Connection refused (error 111)
08:04:05.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:05.148208 24527 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.218.125" port: 44351 } attempt: 6
08:04:05.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:05.181129 24527 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.218.125:44351: Network error: Client connection negotiation failed: client connection to 127.23.218.125:44351: connect: Connection refused (error 111)
08:04:06.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:06.241660 24527 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.218.125" port: 44351 } attempt: 7
08:04:06.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:06.278986 24527 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.218.125:44351: Network error: Client connection negotiation failed: client connection to 127.23.218.125:44351: connect: Connection refused (error 111)
08:04:06.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:05.771421 24541 debug-util.cc:398] Leaking SignalData structure 0x7b0800028d00 after lost signal to thread 24536
08:04:06.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:06.429653 24536 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 2.059s user 0.607s sys 1.183s
08:04:06.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:04.374809 24543 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:04:06.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:06.430217 24536 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 2.059s user 0.607s sys 1.183s
08:04:06.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:06.433473 24545 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:04:06.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:06.435675 24544 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2063 milliseconds
08:04:06.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:06.435683 24536 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
08:04:06.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:06.437053 24536 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:04:06.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:06.440217 24536 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:04:06.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:06.441672 24536 hybrid_clock.cc:648] HybridClock initialized: now 1777536246441615 us; error 67 us; skew 500 ppm
08:04:06.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:06.442760 24536 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:04:06.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 30 08:04:06 dist-test-slave-h4q8 krb5kdc[24437](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777536246, etypes {rep=17 tkt=17 ses=17}, kudu/127.23.218.125@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:04:06.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:06.449280 24536 init.cc:377] Logged in from keytab as kudu/127.23.218.125@KRBTEST.COM (short username kudu)
08:04:06.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:06.455727 24536 webserver.cc:492] Webserver started at http://127.23.218.125:41921/ using document root <none> and password file <none>
08:04:06.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:06.456718 24536 fs_manager.cc:362] Metadata directory not provided
08:04:06.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:06.456964 24536 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:04:06.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:06.457562 24536 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
08:04:06.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:06.461935 24536 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/master-1/data/instance:
08:04:06.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e1b4e8ad29a5471292df64bfbbcf85e3"
08:04:06.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-04-30 08:04:06 on dist-test-slave-h4q8"
08:04:06.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:06.463186 24536 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/master-1/wal/instance:
08:04:06.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e1b4e8ad29a5471292df64bfbbcf85e3"
08:04:06.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-04-30 08:04:06 on dist-test-slave-h4q8"
08:04:06.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:06.472729 24536 fs_manager.cc:696] Time spent creating directory manager: real 0.009s user 0.005s sys 0.005s
08:04:06.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:06.479588 24563 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:04:06.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:06.480811 24536 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.002s sys 0.002s
08:04:06.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:06.481174 24536 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/master-1/data,/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/master-1/wal
08:04:06.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e1b4e8ad29a5471292df64bfbbcf85e3"
08:04:06.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-04-30 08:04:06 on dist-test-slave-h4q8"
08:04:06.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:06.481540 24536 fs_report.cc:389] FS layout report
08:04:06.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:04:06.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/master-1/wal
08:04:06.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/master-1/wal
08:04:06.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/master-1/data/data
08:04:06.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:04:06.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:04:06.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:04:06.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:04:06.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:04:06.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:04:06.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:04:06.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:04:06.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:04:06.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:04:06.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:04:06.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:06.553251 24536 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:04:06.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:06.559733 24536 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:04:06.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:06.560241 24536 kserver.cc:163] Server-wide thread pool size limit: 3276
08:04:06.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:06.668280 24536 rpc_server.cc:307] RPC server started. Bound to: 127.23.218.125:44351
08:04:06.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:06.668318 24614 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.23.218.125:44351 every 8 connection(s)
08:04:06.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:06.671402 24536 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/master-1/data/info.pb
08:04:06.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:06.675185 24425 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskf7pa4T/build/tsan/bin/kudu as pid 24536
08:04:06.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:06.675654 24425 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/master-1/wal/instance
08:04:06.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:06.681330 24615 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
08:04:06.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:06.697810 24615 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.218.126" port: 42861 } has no permanent_uuid. Determining permanent_uuid...
08:04:06.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.23.218.124@KRBTEST.COM; defaulting to no policy
08:04:06.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:04:06.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.23.218.124@KRBTEST.COM" created.
08:04:06.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 30 08:04:06 dist-test-slave-h4q8 krb5kdc[24437](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777536246, etypes {rep=17 tkt=17 ses=17}, kudu/127.23.218.125@KRBTEST.COM for kudu/127.23.218.126@KRBTEST.COM
08:04:06.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:06.738415 24615 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.218.125" port: 44351 } has no permanent_uuid. Determining permanent_uuid...
08:04:06.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:04:06.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.23.218.124 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/krb5kdc/kudu.keytab.
08:04:06.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.23.218.124 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/krb5kdc/kudu.keytab.
08:04:06.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 30 08:04:06 dist-test-slave-h4q8 krb5kdc[24437](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777536246, etypes {rep=17 tkt=17 ses=17}, kudu/127.23.218.125@KRBTEST.COM for kudu/127.23.218.125@KRBTEST.COM
08:04:06.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.23.218.124@KRBTEST.COM; defaulting to no policy
08:04:06.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:04:06.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.23.218.124@KRBTEST.COM" created.
08:04:06.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:06.774287 24615 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.218.124" port: 32793 } has no permanent_uuid. Determining permanent_uuid...
08:04:06.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:06.777516 24564 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.23.218.124:32793: connect: Connection refused (error 111)
08:04:06.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:06.787034 24615 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.218.124:32793: Network error: Client connection negotiation failed: client connection to 127.23.218.124:32793: connect: Connection refused (error 111)
08:04:06.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:04:06.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.23.218.124 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/krb5kdc/kudu.keytab.
08:04:06.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.23.218.124 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/krb5kdc/kudu.keytab.
08:04:06.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:06.799082 24425 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskf7pa4T/build/tsan/bin/kudu
08:04:06.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskf7pa4T/build/tsan/bin/kudu
08:04:06.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/master-2/wal
08:04:06.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/master-2/data
08:04:06.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:04:06.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:04:06.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:04:06.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:04:06.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:04:06.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:04:06.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/master-2/logs
08:04:06.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/master-2/data/info.pb
08:04:06.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:04:06.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:04:06.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:04:06.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:04:06.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:04:06.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:04:06.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:04:06.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:04:06.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:04:06.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:04:06.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:04:06.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:04:06.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.218.124:32793
08:04:06.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.218.124
08:04:06.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:04:06.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/krb5kdc/kudu.keytab
08:04:06.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.23.218.124
08:04:06.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:04:06.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
08:04:06.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
08:04:06.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
08:04:06.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.218.84:37547
08:04:06.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:04:06.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:04:06.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:04:06.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:04:06.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.218.126:42861,127.23.218.125:44351,127.23.218.124:32793
08:04:06.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:04:06.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43537/jwks.json
08:04:06.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
08:04:06.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
08:04:06.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:06.836608 24615 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.218.124" port: 32793 } attempt: 1
08:04:06.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:06.841814 24615 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.218.124:32793: Network error: Client connection negotiation failed: client connection to 127.23.218.124:32793: connect: Connection refused (error 111)
08:04:06.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:06.910234 24615 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.218.124" port: 32793 } attempt: 2
08:04:06.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:06.915161 24615 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.218.124:32793: Network error: Client connection negotiation failed: client connection to 127.23.218.124:32793: connect: Connection refused (error 111)
08:04:07.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:07.006615 24615 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.218.124" port: 32793 } attempt: 3
08:04:07.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:07.011890 24615 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.218.124:32793: Network error: Client connection negotiation failed: client connection to 127.23.218.124:32793: connect: Connection refused (error 111)
08:04:07.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:07.139624 24626 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
08:04:07.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:07.140290 24626 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:04:07.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:07.141036 24626 flags.cc:432] Enabled unsafe flag: --never_fsync=true
08:04:07.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:07.155364 24615 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.218.124" port: 32793 } attempt: 4
08:04:07.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:07.159808 24615 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.218.124:32793: Network error: Client connection negotiation failed: client connection to 127.23.218.124:32793: connect: Connection refused (error 111)
08:04:07.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:07.175577 24626 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
08:04:07.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:07.175884 24626 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
08:04:07.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:07.176328 24626 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
08:04:07.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:07.176651 24626 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
08:04:07.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:07.176934 24626 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
08:04:07.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:07.177227 24626 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
08:04:07.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:07.177488 24626 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:43537/jwks.json
08:04:07.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:07.215220 24626 master_runner.cc:386] Master server non-default flags:
08:04:07.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:04:07.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.218.84:37547
08:04:07.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:04:07.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:04:07.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/master-2/data
08:04:07.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/master-2/wal
08:04:07.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
08:04:07.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:04:07.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.218.126:42861,127.23.218.125:44351,127.23.218.124:32793
08:04:07.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/krb5kdc/kudu.keytab
08:04:07.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.23.218.124
08:04:07.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:04:07.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:04:07.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:04:07.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.218.124:32793
08:04:07.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:04:07.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:04:07.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:04:07.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43537/jwks.json
08:04:07.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:04:07.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
08:04:07.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
08:04:07.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:04:07.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:04:07.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/master-2/data/info.pb
08:04:07.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.218.124
08:04:07.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:04:07.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:04:07.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:04:07.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:04:07.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:04:07.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:04:07.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/master-2/logs
08:04:07.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:04:07.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:04:07.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:04:07.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:04:07.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:04:07.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 8537ed5bb5e0e6ac1e481ff961e6b9d08cce7671
08:04:07.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:04:07.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 30 Apr 2026 07:43:22 UTC on e7f111948823
08:04:07.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11675
08:04:07.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:04:07.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:07.216862 24626 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:04:07.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:07.218123 24626 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:04:07.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:07.230854 24632 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:04:07.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:07.459241 24615 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.218.124" port: 32793 } attempt: 5
08:04:07.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:07.465582 24615 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.218.124:32793: Network error: Client connection negotiation failed: client connection to 127.23.218.124:32793: connect: Connection refused (error 111)
08:04:08.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:08.013299 24615 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.218.124" port: 32793 } attempt: 6
08:04:08.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:08.027048 24615 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.218.124:32793: Network error: Client connection negotiation failed: client connection to 127.23.218.124:32793: connect: Connection refused (error 111)
08:04:08.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:08.369563 24527 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.218.125" port: 44351 } attempt: 8
08:04:08.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 30 08:04:08 dist-test-slave-h4q8 krb5kdc[24437](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777536243, etypes {rep=17 tkt=17 ses=17}, kudu/127.23.218.126@KRBTEST.COM for kudu/127.23.218.125@KRBTEST.COM
08:04:08.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:08.414073 24527 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.218.124" port: 32793 } has no permanent_uuid. Determining permanent_uuid...
08:04:08.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:08.422933 24527 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.218.124:32793: Network error: Client connection negotiation failed: client connection to 127.23.218.124:32793: connect: Connection refused (error 111)
08:04:08.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:08.488433 24527 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.218.124" port: 32793 } attempt: 1
08:04:08.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:08.504858 24527 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.218.124:32793: Network error: Client connection negotiation failed: client connection to 127.23.218.124:32793: connect: Connection refused (error 111)
08:04:08.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:08.558502 24527 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.218.124" port: 32793 } attempt: 2
08:04:08.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:08.566653 24527 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.218.124:32793: Network error: Client connection negotiation failed: client connection to 127.23.218.124:32793: connect: Connection refused (error 111)
08:04:08.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:07.232293 24633 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:04:08.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:07.237771 24635 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:04:08.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:08.607240 24634 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1372 milliseconds
08:04:08.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:08.607355 24626 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
08:04:08.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:08.608711 24626 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:04:08.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:08.611542 24626 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:04:08.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:08.612983 24626 hybrid_clock.cc:648] HybridClock initialized: now 1777536248612942 us; error 42 us; skew 500 ppm
08:04:08.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:08.613997 24626 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:04:08.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 30 08:04:08 dist-test-slave-h4q8 krb5kdc[24437](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777536248, etypes {rep=17 tkt=17 ses=17}, kudu/127.23.218.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:04:08.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:08.621507 24626 init.cc:377] Logged in from keytab as kudu/127.23.218.124@KRBTEST.COM (short username kudu)
08:04:08.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:08.627449 24626 webserver.cc:492] Webserver started at http://127.23.218.124:40583/ using document root <none> and password file <none>
08:04:08.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:08.628463 24626 fs_manager.cc:362] Metadata directory not provided
08:04:08.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:08.628671 24626 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:04:08.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:08.629081 24626 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
08:04:08.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:08.633353 24626 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/master-2/data/instance:
08:04:08.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "24998cc13d3f4ba594cef4e87999214f"
08:04:08.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-04-30 08:04:08 on dist-test-slave-h4q8"
08:04:08.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:08.634485 24626 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/master-2/wal/instance:
08:04:08.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "24998cc13d3f4ba594cef4e87999214f"
08:04:08.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-04-30 08:04:08 on dist-test-slave-h4q8"
08:04:08.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:08.642509 24626 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.007s sys 0.001s
08:04:08.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:08.643254 24527 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.218.124" port: 32793 } attempt: 3
08:04:08.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:08.648021 24527 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.218.124:32793: Network error: Client connection negotiation failed: client connection to 127.23.218.124:32793: connect: Connection refused (error 111)
08:04:08.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:08.649534 24649 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:04:08.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:08.650754 24626 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.001s
08:04:08.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:08.651130 24626 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/master-2/data,/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/master-2/wal
08:04:08.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "24998cc13d3f4ba594cef4e87999214f"
08:04:08.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-04-30 08:04:08 on dist-test-slave-h4q8"
08:04:08.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:08.651489 24626 fs_report.cc:389] FS layout report
08:04:08.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:04:08.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/master-2/wal
08:04:08.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/master-2/wal
08:04:08.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/master-2/data/data
08:04:08.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:04:08.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:04:08.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:04:08.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:04:08.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:04:08.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:04:08.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:04:08.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:04:08.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:04:08.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:04:08.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:04:08.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:08.701851 24626 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:04:08.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:08.711995 24626 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:04:08.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:08.713007 24626 kserver.cc:163] Server-wide thread pool size limit: 3276
08:04:08.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:08.803495 24527 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.218.124" port: 32793 } attempt: 4
08:04:08.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:08.808703 24527 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.218.124:32793: Network error: Client connection negotiation failed: client connection to 127.23.218.124:32793: connect: Connection refused (error 111)
08:04:08.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:08.830719 24626 rpc_server.cc:307] RPC server started. Bound to: 127.23.218.124:32793
08:04:08.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:08.830863 24700 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.23.218.124:32793 every 8 connection(s)
08:04:08.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:08.833889 24626 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/master-2/data/info.pb
08:04:08.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:08.843400 24425 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskf7pa4T/build/tsan/bin/kudu as pid 24626
08:04:08.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:08.843876 24425 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/master-2/wal/instance
08:04:08.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:08.844157 24701 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
08:04:08.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:08.864378 24701 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.218.126" port: 42861 } has no permanent_uuid. Determining permanent_uuid...
08:04:08.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 30 08:04:08 dist-test-slave-h4q8 krb5kdc[24437](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777536248, etypes {rep=17 tkt=17 ses=17}, kudu/127.23.218.124@KRBTEST.COM for kudu/127.23.218.126@KRBTEST.COM
08:04:08.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 30 08:04:08 dist-test-slave-h4q8 krb5kdc[24437](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777536241, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.23.218.126@KRBTEST.COM
08:04:08.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:08.917858 24701 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.218.125" port: 44351 } has no permanent_uuid. Determining permanent_uuid...
08:04:08.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 30 08:04:08 dist-test-slave-h4q8 krb5kdc[24437](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777536241, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.23.218.125@KRBTEST.COM
08:04:08.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 30 08:04:08 dist-test-slave-h4q8 krb5kdc[24437](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777536248, etypes {rep=17 tkt=17 ses=17}, kudu/127.23.218.124@KRBTEST.COM for kudu/127.23.218.125@KRBTEST.COM
08:04:08.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:08.947389 24701 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.218.124" port: 32793 } has no permanent_uuid. Determining permanent_uuid...
08:04:08.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 30 08:04:08 dist-test-slave-h4q8 krb5kdc[24437](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777536248, etypes {rep=17 tkt=17 ses=17}, kudu/127.23.218.124@KRBTEST.COM for kudu/127.23.218.124@KRBTEST.COM
08:04:08.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 30 08:04:08 dist-test-slave-h4q8 krb5kdc[24437](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777536241, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.23.218.124@KRBTEST.COM
08:04:08.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:08.982617 24701 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 24998cc13d3f4ba594cef4e87999214f: Bootstrap starting.
08:04:08.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:08.989125 24701 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 24998cc13d3f4ba594cef4e87999214f: Neither blocks nor log segments found. Creating new log.
08:04:08.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:08.990901 24701 log.cc:826] T 00000000000000000000000000000000 P 24998cc13d3f4ba594cef4e87999214f: Log is configured to *not* fsync() on all Append() calls
08:04:08.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:08.995806 24701 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 24998cc13d3f4ba594cef4e87999214f: No bootstrap required, opened a new log
08:04:09.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:09.016094 24701 raft_consensus.cc:359] T 00000000000000000000000000000000 P 24998cc13d3f4ba594cef4e87999214f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "85f6c126e6444b7bae846a0c871b0e18" member_type: VOTER last_known_addr { host: "127.23.218.126" port: 42861 } } peers { permanent_uuid: "e1b4e8ad29a5471292df64bfbbcf85e3" member_type: VOTER last_known_addr { host: "127.23.218.125" port: 44351 } } peers { permanent_uuid: "24998cc13d3f4ba594cef4e87999214f" member_type: VOTER last_known_addr { host: "127.23.218.124" port: 32793 } }
08:04:09.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:09.016790 24701 raft_consensus.cc:385] T 00000000000000000000000000000000 P 24998cc13d3f4ba594cef4e87999214f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:04:09.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:09.017092 24701 raft_consensus.cc:740] T 00000000000000000000000000000000 P 24998cc13d3f4ba594cef4e87999214f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 24998cc13d3f4ba594cef4e87999214f, State: Initialized, Role: FOLLOWER
08:04:09.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:09.018203 24701 consensus_queue.cc:260] T 00000000000000000000000000000000 P 24998cc13d3f4ba594cef4e87999214f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "85f6c126e6444b7bae846a0c871b0e18" member_type: VOTER last_known_addr { host: "127.23.218.126" port: 42861 } } peers { permanent_uuid: "e1b4e8ad29a5471292df64bfbbcf85e3" member_type: VOTER last_known_addr { host: "127.23.218.125" port: 44351 } } peers { permanent_uuid: "24998cc13d3f4ba594cef4e87999214f" member_type: VOTER last_known_addr { host: "127.23.218.124" port: 32793 } }
08:04:09.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:09.021618 24712 sys_catalog.cc:455] T 00000000000000000000000000000000 P 24998cc13d3f4ba594cef4e87999214f [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "85f6c126e6444b7bae846a0c871b0e18" member_type: VOTER last_known_addr { host: "127.23.218.126" port: 42861 } } peers { permanent_uuid: "e1b4e8ad29a5471292df64bfbbcf85e3" member_type: VOTER last_known_addr { host: "127.23.218.125" port: 44351 } } peers { permanent_uuid: "24998cc13d3f4ba594cef4e87999214f" member_type: VOTER last_known_addr { host: "127.23.218.124" port: 32793 } } }
08:04:09.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:09.022291 24712 sys_catalog.cc:458] T 00000000000000000000000000000000 P 24998cc13d3f4ba594cef4e87999214f [sys.catalog]: This master's current role is: FOLLOWER
08:04:09.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:09.023520 24701 sys_catalog.cc:565] T 00000000000000000000000000000000 P 24998cc13d3f4ba594cef4e87999214f [sys.catalog]: configured and running, proceeding with master startup.
08:04:09.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:09.054031 24723 catalog_manager.cc:1568] T 00000000000000000000000000000000 P 24998cc13d3f4ba594cef4e87999214f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:04:09.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:09.057354 24723 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:04:09.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:09.049307 24626 master_runner.cc:186] Error getting master registration for 127.23.218.126:42861: OK, instance_id { permanent_uuid: "85f6c126e6444b7bae846a0c871b0e18" instance_seqno: 1777536243831596 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
08:04:09.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:09.081229 24626 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
08:04:09.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:09.087666 24615 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.218.124" port: 32793 } attempt: 7
08:04:09.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 30 08:04:09 dist-test-slave-h4q8 krb5kdc[24437](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777536246, etypes {rep=17 tkt=17 ses=17}, kudu/127.23.218.125@KRBTEST.COM for kudu/127.23.218.124@KRBTEST.COM
08:04:09.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:09.105180 24527 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.218.124" port: 32793 } attempt: 5
08:04:09.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 30 08:04:09 dist-test-slave-h4q8 krb5kdc[24437](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777536243, etypes {rep=17 tkt=17 ses=17}, kudu/127.23.218.126@KRBTEST.COM for kudu/127.23.218.124@KRBTEST.COM
08:04:09.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:09.132551 24615 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P e1b4e8ad29a5471292df64bfbbcf85e3: Bootstrap starting.
08:04:09.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:09.142807 24615 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P e1b4e8ad29a5471292df64bfbbcf85e3: Neither blocks nor log segments found. Creating new log.
08:04:09.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:09.145140 24615 log.cc:826] T 00000000000000000000000000000000 P e1b4e8ad29a5471292df64bfbbcf85e3: Log is configured to *not* fsync() on all Append() calls
08:04:09.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:09.146826 24527 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 85f6c126e6444b7bae846a0c871b0e18: Bootstrap starting.
08:04:09.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:09.153630 24615 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P e1b4e8ad29a5471292df64bfbbcf85e3: No bootstrap required, opened a new log
08:04:09.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:09.154691 24527 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 85f6c126e6444b7bae846a0c871b0e18: Neither blocks nor log segments found. Creating new log.
08:04:09.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:09.157228 24527 log.cc:826] T 00000000000000000000000000000000 P 85f6c126e6444b7bae846a0c871b0e18: Log is configured to *not* fsync() on all Append() calls
08:04:09.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:09.161867 24527 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 85f6c126e6444b7bae846a0c871b0e18: No bootstrap required, opened a new log
08:04:09.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:09.183583 24527 raft_consensus.cc:359] T 00000000000000000000000000000000 P 85f6c126e6444b7bae846a0c871b0e18 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "85f6c126e6444b7bae846a0c871b0e18" member_type: VOTER last_known_addr { host: "127.23.218.126" port: 42861 } } peers { permanent_uuid: "e1b4e8ad29a5471292df64bfbbcf85e3" member_type: VOTER last_known_addr { host: "127.23.218.125" port: 44351 } } peers { permanent_uuid: "24998cc13d3f4ba594cef4e87999214f" member_type: VOTER last_known_addr { host: "127.23.218.124" port: 32793 } }
08:04:09.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:09.184288 24527 raft_consensus.cc:385] T 00000000000000000000000000000000 P 85f6c126e6444b7bae846a0c871b0e18 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:04:09.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:09.184809 24527 raft_consensus.cc:740] T 00000000000000000000000000000000 P 85f6c126e6444b7bae846a0c871b0e18 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 85f6c126e6444b7bae846a0c871b0e18, State: Initialized, Role: FOLLOWER
08:04:09.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:09.185740 24527 consensus_queue.cc:260] T 00000000000000000000000000000000 P 85f6c126e6444b7bae846a0c871b0e18 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "85f6c126e6444b7bae846a0c871b0e18" member_type: VOTER last_known_addr { host: "127.23.218.126" port: 42861 } } peers { permanent_uuid: "e1b4e8ad29a5471292df64bfbbcf85e3" member_type: VOTER last_known_addr { host: "127.23.218.125" port: 44351 } } peers { permanent_uuid: "24998cc13d3f4ba594cef4e87999214f" member_type: VOTER last_known_addr { host: "127.23.218.124" port: 32793 } }
08:04:09.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:09.185822 24615 raft_consensus.cc:359] T 00000000000000000000000000000000 P e1b4e8ad29a5471292df64bfbbcf85e3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "85f6c126e6444b7bae846a0c871b0e18" member_type: VOTER last_known_addr { host: "127.23.218.126" port: 42861 } } peers { permanent_uuid: "e1b4e8ad29a5471292df64bfbbcf85e3" member_type: VOTER last_known_addr { host: "127.23.218.125" port: 44351 } } peers { permanent_uuid: "24998cc13d3f4ba594cef4e87999214f" member_type: VOTER last_known_addr { host: "127.23.218.124" port: 32793 } }
08:04:09.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:09.186631 24615 raft_consensus.cc:385] T 00000000000000000000000000000000 P e1b4e8ad29a5471292df64bfbbcf85e3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:04:09.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:09.186956 24615 raft_consensus.cc:740] T 00000000000000000000000000000000 P e1b4e8ad29a5471292df64bfbbcf85e3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e1b4e8ad29a5471292df64bfbbcf85e3, State: Initialized, Role: FOLLOWER
08:04:09.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:09.188112 24615 consensus_queue.cc:260] T 00000000000000000000000000000000 P e1b4e8ad29a5471292df64bfbbcf85e3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "85f6c126e6444b7bae846a0c871b0e18" member_type: VOTER last_known_addr { host: "127.23.218.126" port: 42861 } } peers { permanent_uuid: "e1b4e8ad29a5471292df64bfbbcf85e3" member_type: VOTER last_known_addr { host: "127.23.218.125" port: 44351 } } peers { permanent_uuid: "24998cc13d3f4ba594cef4e87999214f" member_type: VOTER last_known_addr { host: "127.23.218.124" port: 32793 } }
08:04:09.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:09.189925 24728 sys_catalog.cc:455] T 00000000000000000000000000000000 P 85f6c126e6444b7bae846a0c871b0e18 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "85f6c126e6444b7bae846a0c871b0e18" member_type: VOTER last_known_addr { host: "127.23.218.126" port: 42861 } } peers { permanent_uuid: "e1b4e8ad29a5471292df64bfbbcf85e3" member_type: VOTER last_known_addr { host: "127.23.218.125" port: 44351 } } peers { permanent_uuid: "24998cc13d3f4ba594cef4e87999214f" member_type: VOTER last_known_addr { host: "127.23.218.124" port: 32793 } } }
08:04:09.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:09.190987 24728 sys_catalog.cc:458] T 00000000000000000000000000000000 P 85f6c126e6444b7bae846a0c871b0e18 [sys.catalog]: This master's current role is: FOLLOWER
08:04:09.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:09.192212 24527 sys_catalog.cc:565] T 00000000000000000000000000000000 P 85f6c126e6444b7bae846a0c871b0e18 [sys.catalog]: configured and running, proceeding with master startup.
08:04:09.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:09.192631 24712 raft_consensus.cc:493] T 00000000000000000000000000000000 P 24998cc13d3f4ba594cef4e87999214f [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:04:09.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:09.192610 24729 sys_catalog.cc:455] T 00000000000000000000000000000000 P e1b4e8ad29a5471292df64bfbbcf85e3 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "85f6c126e6444b7bae846a0c871b0e18" member_type: VOTER last_known_addr { host: "127.23.218.126" port: 42861 } } peers { permanent_uuid: "e1b4e8ad29a5471292df64bfbbcf85e3" member_type: VOTER last_known_addr { host: "127.23.218.125" port: 44351 } } peers { permanent_uuid: "24998cc13d3f4ba594cef4e87999214f" member_type: VOTER last_known_addr { host: "127.23.218.124" port: 32793 } } }
08:04:09.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:09.193163 24712 raft_consensus.cc:515] T 00000000000000000000000000000000 P 24998cc13d3f4ba594cef4e87999214f [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "85f6c126e6444b7bae846a0c871b0e18" member_type: VOTER last_known_addr { host: "127.23.218.126" port: 42861 } } peers { permanent_uuid: "e1b4e8ad29a5471292df64bfbbcf85e3" member_type: VOTER last_known_addr { host: "127.23.218.125" port: 44351 } } peers { permanent_uuid: "24998cc13d3f4ba594cef4e87999214f" member_type: VOTER last_known_addr { host: "127.23.218.124" port: 32793 } }
08:04:09.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:09.193550 24729 sys_catalog.cc:458] T 00000000000000000000000000000000 P e1b4e8ad29a5471292df64bfbbcf85e3 [sys.catalog]: This master's current role is: FOLLOWER
08:04:09.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:09.195072 24615 sys_catalog.cc:565] T 00000000000000000000000000000000 P e1b4e8ad29a5471292df64bfbbcf85e3 [sys.catalog]: configured and running, proceeding with master startup.
08:04:09.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:09.197777 24712 leader_election.cc:290] T 00000000000000000000000000000000 P 24998cc13d3f4ba594cef4e87999214f [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 85f6c126e6444b7bae846a0c871b0e18 (127.23.218.126:42861), e1b4e8ad29a5471292df64bfbbcf85e3 (127.23.218.125:44351)
08:04:09.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:09.197117 24502 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "24998cc13d3f4ba594cef4e87999214f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "85f6c126e6444b7bae846a0c871b0e18" is_pre_election: true
08:04:09.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:09.199011 24502 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 85f6c126e6444b7bae846a0c871b0e18 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 24998cc13d3f4ba594cef4e87999214f in term 0.
08:04:09.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:09.199127 24590 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "24998cc13d3f4ba594cef4e87999214f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e1b4e8ad29a5471292df64bfbbcf85e3" is_pre_election: true
08:04:09.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:09.200117 24590 raft_consensus.cc:2468] T 00000000000000000000000000000000 P e1b4e8ad29a5471292df64bfbbcf85e3 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 24998cc13d3f4ba594cef4e87999214f in term 0.
08:04:09.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:09.200438 24651 leader_election.cc:304] T 00000000000000000000000000000000 P 24998cc13d3f4ba594cef4e87999214f [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 24998cc13d3f4ba594cef4e87999214f, 85f6c126e6444b7bae846a0c871b0e18; no voters:
08:04:09.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:09.201423 24712 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 24998cc13d3f4ba594cef4e87999214f [term 0 FOLLOWER]: Leader pre-election won for term 1
08:04:09.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:09.201756 24712 raft_consensus.cc:493] T 00000000000000000000000000000000 P 24998cc13d3f4ba594cef4e87999214f [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
08:04:09.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:09.202078 24712 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 24998cc13d3f4ba594cef4e87999214f [term 0 FOLLOWER]: Advancing to term 1
08:04:09.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:09.211561 24712 raft_consensus.cc:515] T 00000000000000000000000000000000 P 24998cc13d3f4ba594cef4e87999214f [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "85f6c126e6444b7bae846a0c871b0e18" member_type: VOTER last_known_addr { host: "127.23.218.126" port: 42861 } } peers { permanent_uuid: "e1b4e8ad29a5471292df64bfbbcf85e3" member_type: VOTER last_known_addr { host: "127.23.218.125" port: 44351 } } peers { permanent_uuid: "24998cc13d3f4ba594cef4e87999214f" member_type: VOTER last_known_addr { host: "127.23.218.124" port: 32793 } }
08:04:09.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:09.222329 24655 tablet.cc:2404] T 00000000000000000000000000000000 P 24998cc13d3f4ba594cef4e87999214f: Can't schedule compaction. Clean time has not been advanced past its initial value.
08:04:09.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:09.226070 24502 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "24998cc13d3f4ba594cef4e87999214f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "85f6c126e6444b7bae846a0c871b0e18"
08:04:09.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:09.226852 24502 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 85f6c126e6444b7bae846a0c871b0e18 [term 0 FOLLOWER]: Advancing to term 1
08:04:09.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:09.241163 24590 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "24998cc13d3f4ba594cef4e87999214f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e1b4e8ad29a5471292df64bfbbcf85e3"
08:04:09.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:09.242758 24590 raft_consensus.cc:3060] T 00000000000000000000000000000000 P e1b4e8ad29a5471292df64bfbbcf85e3 [term 0 FOLLOWER]: Advancing to term 1
08:04:09.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:09.254007 24712 leader_election.cc:290] T 00000000000000000000000000000000 P 24998cc13d3f4ba594cef4e87999214f [CANDIDATE]: Term 1 election: Requested vote from peers 85f6c126e6444b7bae846a0c871b0e18 (127.23.218.126:42861), e1b4e8ad29a5471292df64bfbbcf85e3 (127.23.218.125:44351)
08:04:09.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:09.264480 24502 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 85f6c126e6444b7bae846a0c871b0e18 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 24998cc13d3f4ba594cef4e87999214f in term 1.
08:04:09.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:09.266812 24651 leader_election.cc:304] T 00000000000000000000000000000000 P 24998cc13d3f4ba594cef4e87999214f [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 24998cc13d3f4ba594cef4e87999214f, 85f6c126e6444b7bae846a0c871b0e18; no voters:
08:04:09.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:09.268357 24712 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 24998cc13d3f4ba594cef4e87999214f [term 1 FOLLOWER]: Leader election won for term 1
08:04:09.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:09.253304 24590 raft_consensus.cc:2468] T 00000000000000000000000000000000 P e1b4e8ad29a5471292df64bfbbcf85e3 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 24998cc13d3f4ba594cef4e87999214f in term 1.
08:04:09.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:09.275292 24712 raft_consensus.cc:697] T 00000000000000000000000000000000 P 24998cc13d3f4ba594cef4e87999214f [term 1 LEADER]: Becoming Leader. State: Replica: 24998cc13d3f4ba594cef4e87999214f, State: Running, Role: LEADER
08:04:09.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:09.277333 24712 consensus_queue.cc:237] T 00000000000000000000000000000000 P 24998cc13d3f4ba594cef4e87999214f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "85f6c126e6444b7bae846a0c871b0e18" member_type: VOTER last_known_addr { host: "127.23.218.126" port: 42861 } } peers { permanent_uuid: "e1b4e8ad29a5471292df64bfbbcf85e3" member_type: VOTER last_known_addr { host: "127.23.218.125" port: 44351 } } peers { permanent_uuid: "24998cc13d3f4ba594cef4e87999214f" member_type: VOTER last_known_addr { host: "127.23.218.124" port: 32793 } }
08:04:09.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:09.284690 24481 tablet.cc:2404] T 00000000000000000000000000000000 P 85f6c126e6444b7bae846a0c871b0e18: Can't schedule compaction. Clean time has not been advanced past its initial value.
08:04:09.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:09.302145 24745 catalog_manager.cc:805] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
08:04:09.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:09.318389 24739 sys_catalog.cc:455] T 00000000000000000000000000000000 P 24998cc13d3f4ba594cef4e87999214f [sys.catalog]: SysCatalogTable state changed. Reason: New leader 24998cc13d3f4ba594cef4e87999214f. Latest consensus state: current_term: 1 leader_uuid: "24998cc13d3f4ba594cef4e87999214f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "85f6c126e6444b7bae846a0c871b0e18" member_type: VOTER last_known_addr { host: "127.23.218.126" port: 42861 } } peers { permanent_uuid: "e1b4e8ad29a5471292df64bfbbcf85e3" member_type: VOTER last_known_addr { host: "127.23.218.125" port: 44351 } } peers { permanent_uuid: "24998cc13d3f4ba594cef4e87999214f" member_type: VOTER last_known_addr { host: "127.23.218.124" port: 32793 } } }
08:04:09.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:09.319157 24739 sys_catalog.cc:458] T 00000000000000000000000000000000 P 24998cc13d3f4ba594cef4e87999214f [sys.catalog]: This master's current role is: LEADER
08:04:09.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:09.321146 24569 tablet.cc:2404] T 00000000000000000000000000000000 P e1b4e8ad29a5471292df64bfbbcf85e3: Can't schedule compaction. Clean time has not been advanced past its initial value.
08:04:09.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:09.331260 24748 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
08:04:09.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:09.305470 24536 master_runner.cc:186] Error getting master registration for 127.23.218.126:42861: OK, instance_id { permanent_uuid: "85f6c126e6444b7bae846a0c871b0e18" instance_seqno: 1777536243831596 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
08:04:09.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:09.340956 24748 catalog_manager.cc:1494] Initializing Kudu cluster ID...
08:04:09.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:09.341480 24536 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
08:04:09.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:09.367331 24590 raft_consensus.cc:1275] T 00000000000000000000000000000000 P e1b4e8ad29a5471292df64bfbbcf85e3 [term 1 FOLLOWER]: Refusing update from remote peer 24998cc13d3f4ba594cef4e87999214f: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
08:04:09.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:09.369550 24739 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 24998cc13d3f4ba594cef4e87999214f [LEADER]: Connected to new peer: Peer: permanent_uuid: "e1b4e8ad29a5471292df64bfbbcf85e3" member_type: VOTER last_known_addr { host: "127.23.218.125" port: 44351 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
08:04:09.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:09.376163 24502 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 85f6c126e6444b7bae846a0c871b0e18 [term 1 FOLLOWER]: Refusing update from remote peer 24998cc13d3f4ba594cef4e87999214f: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
08:04:09.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:09.378126 24739 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 24998cc13d3f4ba594cef4e87999214f [LEADER]: Connected to new peer: Peer: permanent_uuid: "85f6c126e6444b7bae846a0c871b0e18" member_type: VOTER last_known_addr { host: "127.23.218.126" port: 42861 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
08:04:09.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:09.400485 24728 sys_catalog.cc:455] T 00000000000000000000000000000000 P 85f6c126e6444b7bae846a0c871b0e18 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 24998cc13d3f4ba594cef4e87999214f. Latest consensus state: current_term: 1 leader_uuid: "24998cc13d3f4ba594cef4e87999214f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "85f6c126e6444b7bae846a0c871b0e18" member_type: VOTER last_known_addr { host: "127.23.218.126" port: 42861 } } peers { permanent_uuid: "e1b4e8ad29a5471292df64bfbbcf85e3" member_type: VOTER last_known_addr { host: "127.23.218.125" port: 44351 } } peers { permanent_uuid: "24998cc13d3f4ba594cef4e87999214f" member_type: VOTER last_known_addr { host: "127.23.218.124" port: 32793 } } }
08:04:09.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:09.403194 24728 sys_catalog.cc:458] T 00000000000000000000000000000000 P 85f6c126e6444b7bae846a0c871b0e18 [sys.catalog]: This master's current role is: FOLLOWER
08:04:09.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:09.412045 24752 mvcc.cc:204] Tried to move back new op lower bound from 7280788477389840384 to 7280788477083959296. Current Snapshot: MvccSnapshot[applied={T|T < 7280788477389840384}]
08:04:09.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:09.430555 24729 sys_catalog.cc:455] T 00000000000000000000000000000000 P e1b4e8ad29a5471292df64bfbbcf85e3 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 24998cc13d3f4ba594cef4e87999214f. Latest consensus state: current_term: 1 leader_uuid: "24998cc13d3f4ba594cef4e87999214f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "85f6c126e6444b7bae846a0c871b0e18" member_type: VOTER last_known_addr { host: "127.23.218.126" port: 42861 } } peers { permanent_uuid: "e1b4e8ad29a5471292df64bfbbcf85e3" member_type: VOTER last_known_addr { host: "127.23.218.125" port: 44351 } } peers { permanent_uuid: "24998cc13d3f4ba594cef4e87999214f" member_type: VOTER last_known_addr { host: "127.23.218.124" port: 32793 } } }
08:04:09.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:09.431497 24729 sys_catalog.cc:458] T 00000000000000000000000000000000 P e1b4e8ad29a5471292df64bfbbcf85e3 [sys.catalog]: This master's current role is: FOLLOWER
08:04:09.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:09.437863 24756 mvcc.cc:204] Tried to move back new op lower bound from 7280788477389840384 to 7280788477083959296. Current Snapshot: MvccSnapshot[applied={T|T < 7280788477389840384}]
08:04:09.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:09.441350 24712 sys_catalog.cc:455] T 00000000000000000000000000000000 P 24998cc13d3f4ba594cef4e87999214f [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "24998cc13d3f4ba594cef4e87999214f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "85f6c126e6444b7bae846a0c871b0e18" member_type: VOTER last_known_addr { host: "127.23.218.126" port: 42861 } } peers { permanent_uuid: "e1b4e8ad29a5471292df64bfbbcf85e3" member_type: VOTER last_known_addr { host: "127.23.218.125" port: 44351 } } peers { permanent_uuid: "24998cc13d3f4ba594cef4e87999214f" member_type: VOTER last_known_addr { host: "127.23.218.124" port: 32793 } } }
08:04:09.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:09.442323 24712 sys_catalog.cc:458] T 00000000000000000000000000000000 P 24998cc13d3f4ba594cef4e87999214f [sys.catalog]: This master's current role is: LEADER
08:04:09.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:09.443143 24712 sys_catalog.cc:455] T 00000000000000000000000000000000 P 24998cc13d3f4ba594cef4e87999214f [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "24998cc13d3f4ba594cef4e87999214f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "85f6c126e6444b7bae846a0c871b0e18" member_type: VOTER last_known_addr { host: "127.23.218.126" port: 42861 } } peers { permanent_uuid: "e1b4e8ad29a5471292df64bfbbcf85e3" member_type: VOTER last_known_addr { host: "127.23.218.125" port: 44351 } } peers { permanent_uuid: "24998cc13d3f4ba594cef4e87999214f" member_type: VOTER last_known_addr { host: "127.23.218.124" port: 32793 } } }
08:04:09.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:09.443980 24712 sys_catalog.cc:458] T 00000000000000000000000000000000 P 24998cc13d3f4ba594cef4e87999214f [sys.catalog]: This master's current role is: LEADER
08:04:09.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:09.445964 24728 sys_catalog.cc:455] T 00000000000000000000000000000000 P 85f6c126e6444b7bae846a0c871b0e18 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "24998cc13d3f4ba594cef4e87999214f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "85f6c126e6444b7bae846a0c871b0e18" member_type: VOTER last_known_addr { host: "127.23.218.126" port: 42861 } } peers { permanent_uuid: "e1b4e8ad29a5471292df64bfbbcf85e3" member_type: VOTER last_known_addr { host: "127.23.218.125" port: 44351 } } peers { permanent_uuid: "24998cc13d3f4ba594cef4e87999214f" member_type: VOTER last_known_addr { host: "127.23.218.124" port: 32793 } } }
08:04:09.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:09.446643 24728 sys_catalog.cc:458] T 00000000000000000000000000000000 P 85f6c126e6444b7bae846a0c871b0e18 [sys.catalog]: This master's current role is: FOLLOWER
08:04:09.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:09.456223 24748 catalog_manager.cc:1357] Generated new cluster ID: f8b4d76ecadd440bbbfe796430e93114
08:04:09.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:09.456635 24748 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
08:04:09.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:09.461334 24729 sys_catalog.cc:455] T 00000000000000000000000000000000 P e1b4e8ad29a5471292df64bfbbcf85e3 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "24998cc13d3f4ba594cef4e87999214f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "85f6c126e6444b7bae846a0c871b0e18" member_type: VOTER last_known_addr { host: "127.23.218.126" port: 42861 } } peers { permanent_uuid: "e1b4e8ad29a5471292df64bfbbcf85e3" member_type: VOTER last_known_addr { host: "127.23.218.125" port: 44351 } } peers { permanent_uuid: "24998cc13d3f4ba594cef4e87999214f" member_type: VOTER last_known_addr { host: "127.23.218.124" port: 32793 } } }
08:04:09.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:09.466456 24729 sys_catalog.cc:458] T 00000000000000000000000000000000 P e1b4e8ad29a5471292df64bfbbcf85e3 [sys.catalog]: This master's current role is: FOLLOWER
08:04:09.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:09.466470 24755 catalog_manager.cc:1568] T 00000000000000000000000000000000 P 85f6c126e6444b7bae846a0c871b0e18: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:04:09.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:09.466873 24755 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:04:09.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:09.510149 24748 catalog_manager.cc:1380] Generated new certificate authority record
08:04:09.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:09.512997 24748 catalog_manager.cc:1514] Loading token signing keys...
08:04:09.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:09.553252 24748 catalog_manager.cc:6044] T 00000000000000000000000000000000 P 24998cc13d3f4ba594cef4e87999214f: Generated new TSK 0
08:04:09.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:09.554404 24748 catalog_manager.cc:1524] Initializing in-progress tserver states...
08:04:09.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.23.218.65@KRBTEST.COM; defaulting to no policy
08:04:09.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:04:09.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.23.218.65@KRBTEST.COM" created.
08:04:09.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:04:09.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.23.218.65 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/krb5kdc/kudu.keytab.
08:04:09.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.23.218.65 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/krb5kdc/kudu.keytab.
08:04:09.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.23.218.65@KRBTEST.COM; defaulting to no policy
08:04:09.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:04:09.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.23.218.65@KRBTEST.COM" created.
08:04:09.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:04:09.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.23.218.65 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/krb5kdc/kudu.keytab.
08:04:09.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.23.218.65 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/krb5kdc/kudu.keytab.
08:04:09.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:09.697165 24425 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskf7pa4T/build/tsan/bin/kudu
08:04:09.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskf7pa4T/build/tsan/bin/kudu
08:04:09.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/ts-0/wal
08:04:09.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/ts-0/data
08:04:09.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:04:09.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:04:09.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:04:09.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:04:09.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:04:09.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:04:09.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/ts-0/logs
08:04:09.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/ts-0/data/info.pb
08:04:09.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:04:09.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:04:09.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:04:09.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:04:09.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:04:09.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:04:09.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:04:09.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:04:09.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
08:04:09.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:04:09.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.218.65:0
08:04:09.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.218.65
08:04:09.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.218.65
08:04:09.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:04:09.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.218.126:42861,127.23.218.125:44351,127.23.218.124:32793
08:04:09.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/krb5kdc/kudu.keytab
08:04:09.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.23.218.65
08:04:09.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:04:09.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
08:04:09.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
08:04:09.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
08:04:09.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.218.84:37547
08:04:09.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:04:09.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:04:09.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:04:09.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false with env {KRB5CCNAME=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
08:04:09.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:09.767503 24127 debug-util.cc:398] Leaking SignalData structure 0x7b08000b5ac0 after lost signal to thread 23999
08:04:09.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:09.769399 24127 debug-util.cc:398] Leaking SignalData structure 0x7b0800042460 after lost signal to thread 24130
08:04:10.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:10.126641 24769 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
08:04:10.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:10.127640 24769 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:04:10.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:10.128916 24769 flags.cc:432] Enabled unsafe flag: --never_fsync=true
08:04:10.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:10.170581 24769 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
08:04:10.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:10.171728 24769 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.23.218.65
08:04:10.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:10.212181 24769 tablet_server_runner.cc:78] Tablet server non-default flags:
08:04:10.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:04:10.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.218.84:37547
08:04:10.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:04:10.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:04:10.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/ts-0/data
08:04:10.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/ts-0/wal
08:04:10.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/krb5kdc/kudu.keytab
08:04:10.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.23.218.65
08:04:10.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:04:10.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:04:10.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.218.65:0
08:04:10.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:04:10.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:04:10.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
08:04:10.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
08:04:10.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:04:10.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:04:10.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/ts-0/data/info.pb
08:04:10.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.218.65
08:04:10.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:04:10.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.218.126:42861,127.23.218.125:44351,127.23.218.124:32793
08:04:10.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:04:10.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:04:10.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:04:10.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:04:10.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:04:10.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.218.65
08:04:10.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/ts-0/logs
08:04:10.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:04:10.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:04:10.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:04:10.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
08:04:10.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:04:10.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 8537ed5bb5e0e6ac1e481ff961e6b9d08cce7671
08:04:10.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:04:10.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 30 Apr 2026 07:43:22 UTC on e7f111948823
08:04:10.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11675
08:04:10.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:04:10.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:10.213864 24769 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:04:10.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:10.215179 24769 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:04:10.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:10.229997 24775 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:04:10.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:10.352250 24745 catalog_manager.cc:1269] Loaded cluster ID: f8b4d76ecadd440bbbfe796430e93114
08:04:10.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:10.361896 24745 catalog_manager.cc:1562] T 00000000000000000000000000000000 P e1b4e8ad29a5471292df64bfbbcf85e3: loading cluster ID for follower catalog manager: success
08:04:10.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:10.381474 24745 catalog_manager.cc:1584] T 00000000000000000000000000000000 P e1b4e8ad29a5471292df64bfbbcf85e3: acquiring CA information for follower catalog manager: success
08:04:10.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:10.418757 24745 catalog_manager.cc:1612] T 00000000000000000000000000000000 P e1b4e8ad29a5471292df64bfbbcf85e3: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
08:04:10.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:10.474750 24755 catalog_manager.cc:1269] Loaded cluster ID: f8b4d76ecadd440bbbfe796430e93114
08:04:10.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:10.475591 24755 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 85f6c126e6444b7bae846a0c871b0e18: loading cluster ID for follower catalog manager: success
08:04:10.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:10.499751 24755 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 85f6c126e6444b7bae846a0c871b0e18: acquiring CA information for follower catalog manager: success
08:04:10.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:10.537353 24755 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 85f6c126e6444b7bae846a0c871b0e18: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
08:04:12.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:11.631711 24774 debug-util.cc:398] Leaking SignalData structure 0x7b0800027120 after lost signal to thread 24769
08:04:12.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:12.802071 24774 kernel_stack_watchdog.cc:198] Thread 24769 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 395ms:
08:04:12.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
08:04:12.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
08:04:12.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
08:04:12.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
08:04:12.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:12.805318 24778 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:04:12.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:10.234097 24776 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:04:12.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:12.811517 24777 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection timed out after 2577 milliseconds
08:04:12.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:12.809609 24769 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 2.575s user 0.380s sys 0.714s
08:04:12.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:12.812459 24769 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 2.579s user 0.380s sys 0.714s
08:04:12.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:12.812758 24769 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
08:04:12.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:12.814121 24769 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:04:12.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:12.828848 24769 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:04:12.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:12.833117 24769 hybrid_clock.cc:648] HybridClock initialized: now 1777536252832396 us; error 703 us; skew 500 ppm
08:04:12.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:12.834342 24769 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:04:12.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 30 08:04:12 dist-test-slave-h4q8 krb5kdc[24437](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777536252, etypes {rep=17 tkt=17 ses=17}, kudu/127.23.218.65@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:04:12.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:12.843849 24769 init.cc:377] Logged in from keytab as kudu/127.23.218.65@KRBTEST.COM (short username kudu)
08:04:12.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:12.894111 24769 webserver.cc:492] Webserver started at http://127.23.218.65:33655/ using document root <none> and password file <none>
08:04:12.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:12.896153 24769 fs_manager.cc:362] Metadata directory not provided
08:04:12.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:12.896687 24769 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:04:12.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:12.897428 24769 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
08:04:12.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:12.904665 24769 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/ts-0/data/instance:
08:04:12.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b5d001440eba49d5b0265effe47d3c2c"
08:04:12.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-04-30 08:04:12 on dist-test-slave-h4q8"
08:04:12.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:12.906632 24769 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/ts-0/wal/instance:
08:04:12.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b5d001440eba49d5b0265effe47d3c2c"
08:04:12.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-04-30 08:04:12 on dist-test-slave-h4q8"
08:04:12.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:12.940200 24769 fs_manager.cc:696] Time spent creating directory manager: real 0.022s user 0.015s sys 0.003s
08:04:12.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:12.958742 24789 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:04:12.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:12.960603 24769 fs_manager.cc:730] Time spent opening block manager: real 0.016s user 0.013s sys 0.003s
08:04:12.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:12.961166 24769 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/ts-0/data,/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/ts-0/wal
08:04:12.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b5d001440eba49d5b0265effe47d3c2c"
08:04:12.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-04-30 08:04:12 on dist-test-slave-h4q8"
08:04:12.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:12.961619 24769 fs_report.cc:389] FS layout report
08:04:12.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:04:12.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/ts-0/wal
08:04:12.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/ts-0/wal
08:04:12.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/ts-0/data/data
08:04:12.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:04:12.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:04:12.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:04:12.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:04:12.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:04:12.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:04:12.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:04:12.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:04:12.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:04:12.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:04:12.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:04:13.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:13.147210 24769 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:04:13.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:13.156828 24769 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:04:13.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:13.157548 24769 kserver.cc:163] Server-wide thread pool size limit: 3276
08:04:13.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:13.178375 24769 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
08:04:13.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:13.196525 24769 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
08:04:13.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:13.196848 24769 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
08:04:13.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:13.197134 24769 ts_tablet_manager.cc:616] Registered 0 tablets
08:04:13.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:13.197351 24769 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
08:04:13.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 30 08:04:13 dist-test-slave-h4q8 krb5kdc[24437](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777536253, etypes {rep=17 tkt=17 ses=17}, kudu/127.23.218.126@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:04:13.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:13.669792 24471 init.cc:295] Successfully reacquired a new kerberos TGT
08:04:13.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:13.814370 24769 rpc_server.cc:307] RPC server started. Bound to: 127.23.218.65:33999
08:04:13.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:13.814494 24903 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.23.218.65:33999 every 8 connection(s)
08:04:13.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:13.818065 24769 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/ts-0/data/info.pb
08:04:13.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:13.826124 24425 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskf7pa4T/build/tsan/bin/kudu as pid 24769
08:04:13.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:13.826516 24425 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/ts-0/wal/instance
08:04:13.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 30 08:04:13 dist-test-slave-h4q8 krb5kdc[24437](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777536252, etypes {rep=17 tkt=17 ses=17}, kudu/127.23.218.65@KRBTEST.COM for kudu/127.23.218.124@KRBTEST.COM
08:04:13.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 30 08:04:13 dist-test-slave-h4q8 krb5kdc[24437](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777536252, etypes {rep=17 tkt=17 ses=17}, kudu/127.23.218.65@KRBTEST.COM for kudu/127.23.218.125@KRBTEST.COM
08:04:13.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:13.899672 24904 heartbeater.cc:344] Connected to a master server at 127.23.218.125:44351
08:04:13.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:13.900681 24904 heartbeater.cc:461] Registering TS with master...
08:04:13.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:13.902109 24904 heartbeater.cc:507] Master 127.23.218.125:44351 requested a full tablet report, sending...
08:04:13.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:13.905917 24579 ts_manager.cc:194] Registered new tserver with Master: b5d001440eba49d5b0265effe47d3c2c (127.23.218.65:33999)
08:04:13.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 30 08:04:13 dist-test-slave-h4q8 krb5kdc[24437](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777536252, etypes {rep=17 tkt=17 ses=17}, kudu/127.23.218.65@KRBTEST.COM for kudu/127.23.218.126@KRBTEST.COM
08:04:13.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:13.912429 24905 heartbeater.cc:344] Connected to a master server at 127.23.218.124:32793
08:04:13.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:13.912833 24905 heartbeater.cc:461] Registering TS with master...
08:04:13.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:13.913920 24905 heartbeater.cc:507] Master 127.23.218.124:32793 requested a full tablet report, sending...
08:04:13.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.23.218.66@KRBTEST.COM; defaulting to no policy
08:04:13.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:13.918166 24666 ts_manager.cc:194] Registered new tserver with Master: b5d001440eba49d5b0265effe47d3c2c (127.23.218.65:33999)
08:04:13.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:04:13.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.23.218.66@KRBTEST.COM" created.
08:04:13.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:13.920754 24666 master_service.cc:502] Signed X509 certificate for tserver {username='kudu', principal='kudu/127.23.218.65@KRBTEST.COM'} at 127.23.218.65:37171
08:04:13.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:13.929039 24906 heartbeater.cc:344] Connected to a master server at 127.23.218.126:42861
08:04:13.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:13.929401 24906 heartbeater.cc:461] Registering TS with master...
08:04:13.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:13.930158 24906 heartbeater.cc:507] Master 127.23.218.126:42861 requested a full tablet report, sending...
08:04:13.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:13.932708 24492 ts_manager.cc:194] Registered new tserver with Master: b5d001440eba49d5b0265effe47d3c2c (127.23.218.65:33999)
08:04:13.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:04:13.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.23.218.66 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/krb5kdc/kudu.keytab.
08:04:13.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.23.218.66 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/krb5kdc/kudu.keytab.
08:04:13.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.23.218.66@KRBTEST.COM; defaulting to no policy
08:04:13.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:04:13.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.23.218.66@KRBTEST.COM" created.
08:04:13.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:04:13.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.23.218.66 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/krb5kdc/kudu.keytab.
08:04:13.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.23.218.66 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/krb5kdc/kudu.keytab.
08:04:13.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:13.994735 24425 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskf7pa4T/build/tsan/bin/kudu
08:04:13.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskf7pa4T/build/tsan/bin/kudu
08:04:13.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/ts-1/wal
08:04:13.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/ts-1/data
08:04:13.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:04:13.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:04:13.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:04:13.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:04:13.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:04:13.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:04:13.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/ts-1/logs
08:04:13.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/ts-1/data/info.pb
08:04:13.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:04:13.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:04:13.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:04:13.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:04:13.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:04:13.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:04:13.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:04:13.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:04:13.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
08:04:13.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:04:13.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.218.66:0
08:04:13.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.218.66
08:04:13.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.218.66
08:04:13.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:04:13.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.218.126:42861,127.23.218.125:44351,127.23.218.124:32793
08:04:13.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/krb5kdc/kudu.keytab
08:04:13.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.23.218.66
08:04:13.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:04:13.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
08:04:13.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
08:04:13.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
08:04:13.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.218.84:37547
08:04:13.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:04:13.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:04:13.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:04:13.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false with env {KRB5CCNAME=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
08:04:14.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:14.406003 24918 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
08:04:14.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:14.406467 24918 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:04:14.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:14.406996 24918 flags.cc:432] Enabled unsafe flag: --never_fsync=true
08:04:14.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:14.443444 24918 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
08:04:14.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:14.444602 24918 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.23.218.66
08:04:14.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:14.503923 24918 tablet_server_runner.cc:78] Tablet server non-default flags:
08:04:14.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:04:14.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.218.84:37547
08:04:14.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:04:14.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:04:14.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/ts-1/data
08:04:14.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/ts-1/wal
08:04:14.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/krb5kdc/kudu.keytab
08:04:14.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.23.218.66
08:04:14.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:04:14.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:04:14.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.218.66:0
08:04:14.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:04:14.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:04:14.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
08:04:14.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
08:04:14.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:04:14.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:04:14.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/ts-1/data/info.pb
08:04:14.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.218.66
08:04:14.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:04:14.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.218.126:42861,127.23.218.125:44351,127.23.218.124:32793
08:04:14.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:04:14.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:04:14.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:04:14.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:04:14.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:04:14.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.218.66
08:04:14.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/ts-1/logs
08:04:14.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:04:14.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:04:14.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:04:14.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
08:04:14.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:04:14.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 8537ed5bb5e0e6ac1e481ff961e6b9d08cce7671
08:04:14.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:04:14.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 30 Apr 2026 07:43:22 UTC on e7f111948823
08:04:14.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11675
08:04:14.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:04:14.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:14.505672 24918 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:04:14.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:14.507014 24918 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:04:14.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:14.521037 24925 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:04:14.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:14.926009 24905 heartbeater.cc:499] Master 127.23.218.124:32793 was elected leader, sending a full tablet report...
08:04:15.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:14.524276 24926 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:04:15.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:14.525605 24928 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:04:15.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:15.762140 24927 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
08:04:15.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:15.762178 24918 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
08:04:15.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:15.766705 24918 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:04:15.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:15.770117 24918 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:04:15.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:15.771593 24918 hybrid_clock.cc:648] HybridClock initialized: now 1777536255771556 us; error 45 us; skew 500 ppm
08:04:15.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:15.772533 24918 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:04:15.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 30 08:04:15 dist-test-slave-h4q8 krb5kdc[24437](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777536255, etypes {rep=17 tkt=17 ses=17}, kudu/127.23.218.66@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:04:15.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:15.779036 24918 init.cc:377] Logged in from keytab as kudu/127.23.218.66@KRBTEST.COM (short username kudu)
08:04:15.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:15.791106 24918 webserver.cc:492] Webserver started at http://127.23.218.66:44223/ using document root <none> and password file <none>
08:04:15.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:15.792197 24918 fs_manager.cc:362] Metadata directory not provided
08:04:15.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:15.792464 24918 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:04:15.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:15.793027 24918 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
08:04:15.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:15.797940 24918 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/ts-1/data/instance:
08:04:15.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ce738aaf37fe46cab22a838800c45dbf"
08:04:15.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-04-30 08:04:15 on dist-test-slave-h4q8"
08:04:15.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:15.800042 24918 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/ts-1/wal/instance:
08:04:15.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ce738aaf37fe46cab22a838800c45dbf"
08:04:15.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-04-30 08:04:15 on dist-test-slave-h4q8"
08:04:15.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:15.808279 24918 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.000s sys 0.007s
08:04:15.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:15.816097 24938 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:04:15.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:15.817482 24918 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.005s sys 0.000s
08:04:15.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:15.817939 24918 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/ts-1/data,/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/ts-1/wal
08:04:15.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ce738aaf37fe46cab22a838800c45dbf"
08:04:15.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-04-30 08:04:15 on dist-test-slave-h4q8"
08:04:15.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:15.818397 24918 fs_report.cc:389] FS layout report
08:04:15.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:04:15.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/ts-1/wal
08:04:15.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/ts-1/wal
08:04:15.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/ts-1/data/data
08:04:15.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:04:15.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:04:15.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:04:15.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:04:15.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:04:15.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:04:15.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:04:15.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:04:15.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:04:15.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:04:15.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:04:15.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:15.869230 24918 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:04:15.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:15.875305 24918 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:04:15.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:15.875710 24918 kserver.cc:163] Server-wide thread pool size limit: 3276
08:04:15.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:15.879345 24918 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
08:04:15.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:15.884570 24918 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
08:04:15.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:15.884851 24918 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
08:04:15.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:15.885089 24918 ts_tablet_manager.cc:616] Registered 0 tablets
08:04:15.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:15.885229 24918 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
08:04:16.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:16.048533 24918 rpc_server.cc:307] RPC server started. Bound to: 127.23.218.66:34745
08:04:16.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:16.048650 25050 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.23.218.66:34745 every 8 connection(s)
08:04:16.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:16.051276 24918 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/ts-1/data/info.pb
08:04:16.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:16.055357 24425 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskf7pa4T/build/tsan/bin/kudu as pid 24918
08:04:16.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:16.055856 24425 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/ts-1/wal/instance
08:04:16.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.23.218.67@KRBTEST.COM; defaulting to no policy
08:04:16.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:04:16.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.23.218.67@KRBTEST.COM" created.
08:04:16.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 30 08:04:16 dist-test-slave-h4q8 krb5kdc[24437](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777536255, etypes {rep=17 tkt=17 ses=17}, kudu/127.23.218.66@KRBTEST.COM for kudu/127.23.218.126@KRBTEST.COM
08:04:16.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 30 08:04:16 dist-test-slave-h4q8 krb5kdc[24437](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777536255, etypes {rep=17 tkt=17 ses=17}, kudu/127.23.218.66@KRBTEST.COM for kudu/127.23.218.125@KRBTEST.COM
08:04:16.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 30 08:04:16 dist-test-slave-h4q8 krb5kdc[24437](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777536255, etypes {rep=17 tkt=17 ses=17}, kudu/127.23.218.66@KRBTEST.COM for kudu/127.23.218.124@KRBTEST.COM
08:04:16.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:16.129629 25052 heartbeater.cc:344] Connected to a master server at 127.23.218.124:32793
08:04:16.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:16.130290 25052 heartbeater.cc:461] Registering TS with master...
08:04:16.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:16.132236 25052 heartbeater.cc:507] Master 127.23.218.124:32793 requested a full tablet report, sending...
08:04:16.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:16.135069 24666 ts_manager.cc:194] Registered new tserver with Master: ce738aaf37fe46cab22a838800c45dbf (127.23.218.66:34745)
08:04:16.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:16.136909 24666 master_service.cc:502] Signed X509 certificate for tserver {username='kudu', principal='kudu/127.23.218.66@KRBTEST.COM'} at 127.23.218.66:52255
08:04:16.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:16.140353 25054 heartbeater.cc:344] Connected to a master server at 127.23.218.126:42861
08:04:16.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:16.140679 25054 heartbeater.cc:461] Registering TS with master...
08:04:16.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:04:16.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.23.218.67 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/krb5kdc/kudu.keytab.
08:04:16.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.23.218.67 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/krb5kdc/kudu.keytab.
08:04:16.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:16.141885 25054 heartbeater.cc:507] Master 127.23.218.126:42861 requested a full tablet report, sending...
08:04:16.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:16.149582 24492 ts_manager.cc:194] Registered new tserver with Master: ce738aaf37fe46cab22a838800c45dbf (127.23.218.66:34745)
08:04:16.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:16.149720 25051 heartbeater.cc:344] Connected to a master server at 127.23.218.125:44351
08:04:16.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:16.150552 25051 heartbeater.cc:461] Registering TS with master...
08:04:16.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:16.151208 25051 heartbeater.cc:507] Master 127.23.218.125:44351 requested a full tablet report, sending...
08:04:16.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:16.153358 24579 ts_manager.cc:194] Registered new tserver with Master: ce738aaf37fe46cab22a838800c45dbf (127.23.218.66:34745)
08:04:16.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.23.218.67@KRBTEST.COM; defaulting to no policy
08:04:16.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:04:16.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.23.218.67@KRBTEST.COM" created.
08:04:16.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:04:16.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.23.218.67 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/krb5kdc/kudu.keytab.
08:04:16.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.23.218.67 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/krb5kdc/kudu.keytab.
08:04:16.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:16.185990 24425 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskf7pa4T/build/tsan/bin/kudu
08:04:16.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskf7pa4T/build/tsan/bin/kudu
08:04:16.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/ts-2/wal
08:04:16.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/ts-2/data
08:04:16.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:04:16.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:04:16.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:04:16.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:04:16.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:04:16.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:04:16.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/ts-2/logs
08:04:16.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/ts-2/data/info.pb
08:04:16.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:04:16.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:04:16.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:04:16.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:04:16.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:04:16.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:04:16.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:04:16.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:04:16.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
08:04:16.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:04:16.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.218.67:0
08:04:16.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.218.67
08:04:16.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.218.67
08:04:16.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:04:16.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.218.126:42861,127.23.218.125:44351,127.23.218.124:32793
08:04:16.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/krb5kdc/kudu.keytab
08:04:16.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.23.218.67
08:04:16.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:04:16.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
08:04:16.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
08:04:16.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
08:04:16.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.218.84:37547
08:04:16.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:04:16.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:04:16.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:04:16.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false with env {KRB5CCNAME=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
08:04:16.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 30 08:04:16 dist-test-slave-h4q8 krb5kdc[24437](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777536256, etypes {rep=17 tkt=17 ses=17}, kudu/127.23.218.125@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:04:16.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:16.453950 24559 init.cc:295] Successfully reacquired a new kerberos TGT
08:04:16.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:16.558516 25065 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
08:04:16.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:16.559095 25065 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:04:16.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:16.559808 25065 flags.cc:432] Enabled unsafe flag: --never_fsync=true
08:04:16.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:16.596827 25065 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
08:04:16.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:16.597894 25065 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.23.218.67
08:04:16.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:16.658718 25065 tablet_server_runner.cc:78] Tablet server non-default flags:
08:04:16.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:04:16.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.218.84:37547
08:04:16.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:04:16.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:04:16.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/ts-2/data
08:04:16.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/ts-2/wal
08:04:16.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/krb5kdc/kudu.keytab
08:04:16.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.23.218.67
08:04:16.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:04:16.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:04:16.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.218.67:0
08:04:16.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:04:16.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:04:16.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
08:04:16.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
08:04:16.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:04:16.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:04:16.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/ts-2/data/info.pb
08:04:16.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.218.67
08:04:16.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:04:16.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.218.126:42861,127.23.218.125:44351,127.23.218.124:32793
08:04:16.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:04:16.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:04:16.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:04:16.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:04:16.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:04:16.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.218.67
08:04:16.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/ts-2/logs
08:04:16.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:04:16.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:04:16.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:04:16.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
08:04:16.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:04:16.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 8537ed5bb5e0e6ac1e481ff961e6b9d08cce7671
08:04:16.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:04:16.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 30 Apr 2026 07:43:22 UTC on e7f111948823
08:04:16.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11675
08:04:16.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:04:16.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:16.659999 25065 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:04:16.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:16.661017 25065 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:04:16.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:16.683120 25071 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:04:17.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:17.142869 25052 heartbeater.cc:499] Master 127.23.218.124:32793 was elected leader, sending a full tablet report...
08:04:17.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:17.512638 23928 debug-util.cc:398] Leaking SignalData structure 0x7b08000aaac0 after lost signal to thread 23858
08:04:17.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:17.514005 23928 debug-util.cc:398] Leaking SignalData structure 0x7b08000aa8a0 after lost signal to thread 23931
08:04:18.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:18.078516 25070 debug-util.cc:398] Leaking SignalData structure 0x7b0800027120 after lost signal to thread 25065
08:04:18.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:18.462946 25070 kernel_stack_watchdog.cc:198] Thread 25065 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 391ms:
08:04:18.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
08:04:18.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
08:04:18.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
08:04:18.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
08:04:18.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:18.463904 25065 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.779s user 0.397s sys 0.764s
08:04:18.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:18.464274 25065 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.780s user 0.397s sys 0.764s
08:04:18.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:18.464610 25073 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1777 milliseconds
08:04:18.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:16.686371 25072 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:04:18.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:18.466342 25074 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:04:18.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:18.466323 25065 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
08:04:18.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:18.471016 25065 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:04:18.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:18.473356 25065 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:04:18.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:18.474692 25065 hybrid_clock.cc:648] HybridClock initialized: now 1777536258474652 us; error 56 us; skew 500 ppm
08:04:18.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:18.475543 25065 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:04:18.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 30 08:04:18 dist-test-slave-h4q8 krb5kdc[24437](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777536258, etypes {rep=17 tkt=17 ses=17}, kudu/127.23.218.67@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:04:18.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:18.484226 25065 init.cc:377] Logged in from keytab as kudu/127.23.218.67@KRBTEST.COM (short username kudu)
08:04:18.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:18.493170 25065 webserver.cc:492] Webserver started at http://127.23.218.67:36897/ using document root <none> and password file <none>
08:04:18.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:18.494448 25065 fs_manager.cc:362] Metadata directory not provided
08:04:18.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:18.494711 25065 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:04:18.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:18.495335 25065 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
08:04:18.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:18.502439 25065 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/ts-2/data/instance:
08:04:18.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "eca5d45e7a234d32aceb91dd556eb91f"
08:04:18.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-04-30 08:04:18 on dist-test-slave-h4q8"
08:04:18.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:18.504196 25065 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/ts-2/wal/instance:
08:04:18.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "eca5d45e7a234d32aceb91dd556eb91f"
08:04:18.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-04-30 08:04:18 on dist-test-slave-h4q8"
08:04:18.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:18.515404 25065 fs_manager.cc:696] Time spent creating directory manager: real 0.010s user 0.011s sys 0.000s
08:04:18.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:18.523979 25087 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:04:18.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:18.525511 25065 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.006s sys 0.000s
08:04:18.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:18.525966 25065 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/ts-2/data,/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/ts-2/wal
08:04:18.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "eca5d45e7a234d32aceb91dd556eb91f"
08:04:18.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-04-30 08:04:18 on dist-test-slave-h4q8"
08:04:18.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:18.526433 25065 fs_report.cc:389] FS layout report
08:04:18.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:04:18.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/ts-2/wal
08:04:18.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/ts-2/wal
08:04:18.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/ts-2/data/data
08:04:18.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:04:18.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:04:18.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:04:18.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:04:18.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:04:18.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:04:18.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:04:18.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:04:18.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:04:18.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:04:18.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:04:18.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:18.586033 25065 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:04:18.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:18.593814 25065 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:04:18.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:18.594393 25065 kserver.cc:163] Server-wide thread pool size limit: 3276
08:04:18.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:18.598348 25065 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
08:04:18.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:18.604759 25065 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
08:04:18.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:18.605069 25065 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
08:04:18.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:18.605355 25065 ts_tablet_manager.cc:616] Registered 0 tablets
08:04:18.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:18.605564 25065 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
08:04:18.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 30 08:04:18 dist-test-slave-h4q8 krb5kdc[24437](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777536258, etypes {rep=17 tkt=17 ses=17}, kudu/127.23.218.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:04:18.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:18.629068 24645 init.cc:295] Successfully reacquired a new kerberos TGT
08:04:19.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:19.015300 25065 rpc_server.cc:307] RPC server started. Bound to: 127.23.218.67:35221
08:04:19.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:19.015411 25201 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.23.218.67:35221 every 8 connection(s)
08:04:19.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:19.019843 25065 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/ts-2/data/info.pb
08:04:19.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:19.023223 24425 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskf7pa4T/build/tsan/bin/kudu as pid 25065
08:04:19.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:19.023960 24425 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/ts-2/wal/instance
08:04:19.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 30 08:04:19 dist-test-slave-h4q8 krb5kdc[24437](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777536258, etypes {rep=17 tkt=17 ses=17}, kudu/127.23.218.67@KRBTEST.COM for kudu/127.23.218.126@KRBTEST.COM
08:04:19.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 30 08:04:19 dist-test-slave-h4q8 krb5kdc[24437](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777536258, etypes {rep=17 tkt=17 ses=17}, kudu/127.23.218.67@KRBTEST.COM for kudu/127.23.218.124@KRBTEST.COM
08:04:19.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 30 08:04:19 dist-test-slave-h4q8 krb5kdc[24437](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777536258, etypes {rep=17 tkt=17 ses=17}, kudu/127.23.218.67@KRBTEST.COM for kudu/127.23.218.125@KRBTEST.COM
08:04:19.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:19.111840 25206 heartbeater.cc:344] Connected to a master server at 127.23.218.126:42861
08:04:19.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:19.112349 25206 heartbeater.cc:461] Registering TS with master...
08:04:19.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:19.113778 25206 heartbeater.cc:507] Master 127.23.218.126:42861 requested a full tablet report, sending...
08:04:19.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:19.116768 24492 ts_manager.cc:194] Registered new tserver with Master: eca5d45e7a234d32aceb91dd556eb91f (127.23.218.67:35221)
08:04:19.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:19.125826 25203 heartbeater.cc:344] Connected to a master server at 127.23.218.124:32793
08:04:19.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:19.126240 25203 heartbeater.cc:461] Registering TS with master...
08:04:19.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:19.127277 25203 heartbeater.cc:507] Master 127.23.218.124:32793 requested a full tablet report, sending...
08:04:19.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:19.131187 24666 ts_manager.cc:194] Registered new tserver with Master: eca5d45e7a234d32aceb91dd556eb91f (127.23.218.67:35221)
08:04:19.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:19.131731 25202 heartbeater.cc:344] Connected to a master server at 127.23.218.125:44351
08:04:19.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:19.132120 25202 heartbeater.cc:461] Registering TS with master...
08:04:19.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:19.132826 24666 master_service.cc:502] Signed X509 certificate for tserver {username='kudu', principal='kudu/127.23.218.67@KRBTEST.COM'} at 127.23.218.67:53925
08:04:19.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:19.133105 25202 heartbeater.cc:507] Master 127.23.218.125:44351 requested a full tablet report, sending...
08:04:19.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:19.136503 24579 ts_manager.cc:194] Registered new tserver with Master: eca5d45e7a234d32aceb91dd556eb91f (127.23.218.67:35221)
08:04:19.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:19.145334 24425 external_mini_cluster.cc:949] 3 TS(s) registered with all masters
08:04:19.157 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:04:19.158 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
08:04:19.170 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
08:04:19.173 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
08:04:19.177 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.23.218.126"
port: 42861
}
bound_http_address {
host: "127.23.218.126"
port: 45411
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.23.218.125"
port: 44351
}
bound_http_address {
host: "127.23.218.125"
port: 41921
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.23.218.124"
port: 32793
}
bound_http_address {
host: "127.23.218.124"
port: 40583
}
}
}
08:04:19.178 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
08:04:19.179 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.23.218.65"
port: 33999
}
bound_http_address {
host: "127.23.218.65"
port: 33655
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.23.218.66"
port: 34745
}
bound_http_address {
host: "127.23.218.66"
port: 44223
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.23.218.67"
port: 35221
}
bound_http_address {
host: "127.23.218.67"
port: 36897
}
}
}
08:04:19.180 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
08:04:19.181 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
08:04:19.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 30 08:04:19 dist-test-slave-h4q8 krb5kdc[24437](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777536259, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:04:19.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
08:04:19.216 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:04:19.218 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/krb5kdc/krb5cc
08:04:19.231 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
08:04:19.232 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/krb5kdc/krb5cc
08:04:19.233 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
08:04:19.271 [DEBUG - Time-limited test] (SecurityContext.java:158) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
08:04:19.274 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
08:04:19.274 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
08:04:19.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 30 08:04:19 dist-test-slave-h4q8 krb5kdc[24437](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777536259, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:04:19.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
08:04:19.307 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:04:19.307 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/krb5kdc/krb5cc
08:04:19.309 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
08:04:19.310 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.23.218.125:44351
08:04:19.311 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
08:04:19.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:19.312263 24425 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskf7pa4T/build/tsan/bin/kudu with pid 24536
08:04:19.351 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:04:19.351 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.23.218.126:42861
08:04:19.351 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
08:04:19.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:19.351994 24425 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskf7pa4T/build/tsan/bin/kudu with pid 24454
08:04:19.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:19.362795 24653 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.23.218.125:44351: connect: Connection refused (error 111)
08:04:19.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:19.368628 24653 consensus_peers.cc:597] T 00000000000000000000000000000000 P 24998cc13d3f4ba594cef4e87999214f -> Peer e1b4e8ad29a5471292df64bfbbcf85e3 (127.23.218.125:44351): Couldn't send request to peer e1b4e8ad29a5471292df64bfbbcf85e3. Status: Network error: Client connection negotiation failed: client connection to 127.23.218.125:44351: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
08:04:19.386 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:04:19.386 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.23.218.124:32793
08:04:19.386 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
08:04:19.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:19.386945 24425 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskf7pa4T/build/tsan/bin/kudu with pid 24626
08:04:19.417 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:04:19.417 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.23.218.125:44351
08:04:19.421 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
08:04:19.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:19.426358 24425 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskf7pa4T/build/tsan/bin/kudu
08:04:19.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskf7pa4T/build/tsan/bin/kudu
08:04:19.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/master-1/wal
08:04:19.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/master-1/data
08:04:19.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:04:19.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:04:19.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:04:19.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:04:19.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:04:19.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:04:19.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/master-1/logs
08:04:19.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/master-1/data/info.pb
08:04:19.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:04:19.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:04:19.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:04:19.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:04:19.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:04:19.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:04:19.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:04:19.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:04:19.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:04:19.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:04:19.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:04:19.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:04:19.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.218.125:44351
08:04:19.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.218.125
08:04:19.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41921
08:04:19.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/krb5kdc/kudu.keytab
08:04:19.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.23.218.125
08:04:19.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:04:19.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
08:04:19.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
08:04:19.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
08:04:19.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.218.84:37547
08:04:19.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:04:19.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:04:19.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:04:19.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:04:19.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.218.126:42861,127.23.218.125:44351,127.23.218.124:32793
08:04:19.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:04:19.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43537/jwks.json
08:04:19.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
08:04:19.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
08:04:19.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:19.781569 25218 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
08:04:19.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:19.782325 25218 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:04:19.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:19.783154 25218 flags.cc:432] Enabled unsafe flag: --never_fsync=true
08:04:19.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:19.825595 25218 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
08:04:19.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:19.825985 25218 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
08:04:19.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:19.826421 25218 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
08:04:19.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:19.826783 25218 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
08:04:19.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:19.827123 25218 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
08:04:19.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:19.827454 25218 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
08:04:19.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:19.827709 25218 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:43537/jwks.json
08:04:19.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:19.868750 25218 master_runner.cc:386] Master server non-default flags:
08:04:19.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:04:19.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.218.84:37547
08:04:19.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:04:19.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:04:19.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/master-1/data
08:04:19.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/master-1/wal
08:04:19.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
08:04:19.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:04:19.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.218.126:42861,127.23.218.125:44351,127.23.218.124:32793
08:04:19.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/krb5kdc/kudu.keytab
08:04:19.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.23.218.125
08:04:19.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:04:19.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:04:19.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:04:19.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.218.125:44351
08:04:19.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:04:19.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:04:19.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:04:19.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43537/jwks.json
08:04:19.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:04:19.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
08:04:19.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
08:04:19.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:04:19.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:04:19.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/master-1/data/info.pb
08:04:19.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.218.125
08:04:19.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41921
08:04:19.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:04:19.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:04:19.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:04:19.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:04:19.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:04:19.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/master-1/logs
08:04:19.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:04:19.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:04:19.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:04:19.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:04:19.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:04:19.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 8537ed5bb5e0e6ac1e481ff961e6b9d08cce7671
08:04:19.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:04:19.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 30 Apr 2026 07:43:22 UTC on e7f111948823
08:04:19.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11675
08:04:19.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:04:19.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:19.870559 25218 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:04:19.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:19.872028 25218 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:04:19.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:19.884717 25225 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:04:19.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:19.992584 24906 heartbeater.cc:646] Failed to heartbeat to 127.23.218.126:42861 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.23.218.126:42861: connect: Connection refused (error 111)
08:04:20.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:20.136680 25206 heartbeater.cc:646] Failed to heartbeat to 127.23.218.126:42861 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.23.218.126:42861: connect: Connection refused (error 111)
08:04:20.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:20.136720 25203 heartbeater.cc:499] Master 127.23.218.124:32793 was elected leader, sending a full tablet report...
08:04:20.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:20.187755 25051 heartbeater.cc:646] Failed to heartbeat to 127.23.218.125:44351 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.23.218.125:44351: connect: Connection refused (error 111)
08:04:21.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:21.289345 25224 debug-util.cc:398] Leaking SignalData structure 0x7b0800028d00 after lost signal to thread 25218
08:04:21.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:21.923648 25224 kernel_stack_watchdog.cc:198] Thread 25218 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 399ms:
08:04:21.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
08:04:21.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
08:04:21.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
08:04:21.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
08:04:21.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:19.887025 25226 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:04:21.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:21.925415 25218 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 2.039s user 0.706s sys 1.280s
08:04:21.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:21.925844 25218 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 2.040s user 0.706s sys 1.281s
08:04:21.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:21.926473 25228 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:04:21.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:21.930068 25227 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2037 milliseconds
08:04:21.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:21.930100 25218 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
08:04:21.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:21.932021 25218 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:04:21.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:21.934671 25218 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:04:21.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:21.936185 25218 hybrid_clock.cc:648] HybridClock initialized: now 1777536261936116 us; error 56 us; skew 500 ppm
08:04:21.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:21.937076 25218 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:04:21.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 30 08:04:21 dist-test-slave-h4q8 krb5kdc[24437](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777536261, etypes {rep=17 tkt=17 ses=17}, kudu/127.23.218.125@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:04:21.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:21.943431 25218 init.cc:377] Logged in from keytab as kudu/127.23.218.125@KRBTEST.COM (short username kudu)
08:04:21.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:21.948693 25218 webserver.cc:492] Webserver started at http://127.23.218.125:41921/ using document root <none> and password file <none>
08:04:21.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:21.949591 25218 fs_manager.cc:362] Metadata directory not provided
08:04:21.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:21.949880 25218 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:04:21.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:21.958122 25218 fs_manager.cc:714] Time spent opening directory manager: real 0.005s user 0.000s sys 0.007s
08:04:21.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:21.962597 25254 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:04:21.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:21.963999 25218 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.003s
08:04:21.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:21.964381 25218 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/master-1/data,/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/master-1/wal
08:04:21.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e1b4e8ad29a5471292df64bfbbcf85e3"
08:04:21.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-04-30 08:04:06 on dist-test-slave-h4q8"
08:04:21.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:21.966557 25218 fs_report.cc:389] FS layout report
08:04:21.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:04:21.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/master-1/wal
08:04:21.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/master-1/wal
08:04:21.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/master-1/data/data
08:04:21.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:04:21.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:04:21.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:04:21.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:04:21.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:04:21.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:04:21.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:04:21.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:04:21.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:04:21.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:04:21.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:04:22.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:22.027158 25218 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:04:22.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:22.032882 25218 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:04:22.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:22.033445 25218 kserver.cc:163] Server-wide thread pool size limit: 3276
08:04:22.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:22.123219 25218 rpc_server.cc:307] RPC server started. Bound to: 127.23.218.125:44351
08:04:22.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:22.123351 25306 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.23.218.125:44351 every 8 connection(s)
08:04:22.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:22.127571 25218 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/master-1/data/info.pb
08:04:22.131 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:04:22.131 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.23.218.126:42861
08:04:22.132 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
08:04:22.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:22.130859 24425 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskf7pa4T/build/tsan/bin/kudu as pid 25218
08:04:22.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:22.133951 24425 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskf7pa4T/build/tsan/bin/kudu
08:04:22.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskf7pa4T/build/tsan/bin/kudu
08:04:22.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/master-0/wal
08:04:22.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/master-0/data
08:04:22.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:04:22.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:04:22.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:04:22.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:04:22.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:04:22.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:04:22.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/master-0/logs
08:04:22.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/master-0/data/info.pb
08:04:22.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:04:22.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:04:22.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:04:22.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:04:22.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:04:22.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:04:22.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:04:22.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:04:22.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:04:22.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:04:22.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:04:22.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:04:22.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.218.126:42861
08:04:22.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.218.126
08:04:22.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45411
08:04:22.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/krb5kdc/kudu.keytab
08:04:22.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.23.218.126
08:04:22.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:04:22.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
08:04:22.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
08:04:22.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
08:04:22.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.218.84:37547
08:04:22.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:04:22.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:04:22.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:04:22.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:04:22.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.218.126:42861,127.23.218.125:44351,127.23.218.124:32793
08:04:22.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:04:22.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43537/jwks.json
08:04:22.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
08:04:22.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
08:04:22.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:22.148097 25307 sys_catalog.cc:263] Verifying existing consensus state
08:04:22.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:22.163595 25307 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P e1b4e8ad29a5471292df64bfbbcf85e3: Bootstrap starting.
08:04:22.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:22.225827 25202 heartbeater.cc:344] Connected to a master server at 127.23.218.125:44351
08:04:22.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:22.243340 25307 log.cc:826] T 00000000000000000000000000000000 P e1b4e8ad29a5471292df64bfbbcf85e3: Log is configured to *not* fsync() on all Append() calls
08:04:22.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:22.245877 25051 heartbeater.cc:344] Connected to a master server at 127.23.218.125:44351
08:04:22.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:22.259774 25307 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P e1b4e8ad29a5471292df64bfbbcf85e3: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
08:04:22.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:22.260838 25307 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P e1b4e8ad29a5471292df64bfbbcf85e3: Bootstrap complete.
08:04:22.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:22.282059 25307 raft_consensus.cc:359] T 00000000000000000000000000000000 P e1b4e8ad29a5471292df64bfbbcf85e3 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "85f6c126e6444b7bae846a0c871b0e18" member_type: VOTER last_known_addr { host: "127.23.218.126" port: 42861 } } peers { permanent_uuid: "e1b4e8ad29a5471292df64bfbbcf85e3" member_type: VOTER last_known_addr { host: "127.23.218.125" port: 44351 } } peers { permanent_uuid: "24998cc13d3f4ba594cef4e87999214f" member_type: VOTER last_known_addr { host: "127.23.218.124" port: 32793 } }
08:04:22.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:22.284607 25307 raft_consensus.cc:740] T 00000000000000000000000000000000 P e1b4e8ad29a5471292df64bfbbcf85e3 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: e1b4e8ad29a5471292df64bfbbcf85e3, State: Initialized, Role: FOLLOWER
08:04:22.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:22.285550 25307 consensus_queue.cc:260] T 00000000000000000000000000000000 P e1b4e8ad29a5471292df64bfbbcf85e3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "85f6c126e6444b7bae846a0c871b0e18" member_type: VOTER last_known_addr { host: "127.23.218.126" port: 42861 } } peers { permanent_uuid: "e1b4e8ad29a5471292df64bfbbcf85e3" member_type: VOTER last_known_addr { host: "127.23.218.125" port: 44351 } } peers { permanent_uuid: "24998cc13d3f4ba594cef4e87999214f" member_type: VOTER last_known_addr { host: "127.23.218.124" port: 32793 } }
08:04:22.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:22.288757 25318 sys_catalog.cc:455] T 00000000000000000000000000000000 P e1b4e8ad29a5471292df64bfbbcf85e3 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "85f6c126e6444b7bae846a0c871b0e18" member_type: VOTER last_known_addr { host: "127.23.218.126" port: 42861 } } peers { permanent_uuid: "e1b4e8ad29a5471292df64bfbbcf85e3" member_type: VOTER last_known_addr { host: "127.23.218.125" port: 44351 } } peers { permanent_uuid: "24998cc13d3f4ba594cef4e87999214f" member_type: VOTER last_known_addr { host: "127.23.218.124" port: 32793 } } }
08:04:22.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:22.289916 25318 sys_catalog.cc:458] T 00000000000000000000000000000000 P e1b4e8ad29a5471292df64bfbbcf85e3 [sys.catalog]: This master's current role is: FOLLOWER
08:04:22.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:22.290539 25307 sys_catalog.cc:565] T 00000000000000000000000000000000 P e1b4e8ad29a5471292df64bfbbcf85e3 [sys.catalog]: configured and running, proceeding with master startup.
08:04:22.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:22.321890 25329 catalog_manager.cc:1269] Loaded cluster ID: f8b4d76ecadd440bbbfe796430e93114
08:04:22.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:22.322548 25329 catalog_manager.cc:1562] T 00000000000000000000000000000000 P e1b4e8ad29a5471292df64bfbbcf85e3: loading cluster ID for follower catalog manager: success
08:04:22.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:22.330744 25329 catalog_manager.cc:1584] T 00000000000000000000000000000000 P e1b4e8ad29a5471292df64bfbbcf85e3: acquiring CA information for follower catalog manager: success
08:04:22.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:22.337055 25329 catalog_manager.cc:1612] T 00000000000000000000000000000000 P e1b4e8ad29a5471292df64bfbbcf85e3: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
08:04:22.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:22.318177 25218 master_runner.cc:186] Error getting master registration for 127.23.218.126:42861: Network error: Client connection negotiation failed: client connection to 127.23.218.126:42861: connect: Connection refused (error 111),
08:04:22.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:22.344112 25218 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
08:04:22.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:22.585068 25309 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
08:04:22.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:22.585637 25309 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:04:22.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:22.586633 25309 flags.cc:432] Enabled unsafe flag: --never_fsync=true
08:04:22.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:22.594981 23847 debug-util.cc:398] Leaking SignalData structure 0x7b08000a6fe0 after lost signal to thread 23782
08:04:22.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:22.595737 23847 debug-util.cc:398] Leaking SignalData structure 0x7b0800088f40 after lost signal to thread 23850
08:04:22.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:22.626283 25309 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
08:04:22.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:22.626647 25309 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
08:04:22.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:22.627242 25309 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
08:04:22.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:22.627633 25309 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
08:04:22.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:22.628001 25309 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
08:04:22.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:22.628371 25309 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
08:04:22.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:22.628652 25309 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:43537/jwks.json
08:04:22.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:22.709352 25309 master_runner.cc:386] Master server non-default flags:
08:04:22.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:04:22.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.218.84:37547
08:04:22.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:04:22.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:04:22.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/master-0/data
08:04:22.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/master-0/wal
08:04:22.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
08:04:22.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:04:22.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.218.126:42861,127.23.218.125:44351,127.23.218.124:32793
08:04:22.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/krb5kdc/kudu.keytab
08:04:22.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.23.218.126
08:04:22.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:04:22.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:04:22.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:04:22.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.218.126:42861
08:04:22.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:04:22.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:04:22.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:04:22.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43537/jwks.json
08:04:22.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:04:22.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
08:04:22.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
08:04:22.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:04:22.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:04:22.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/master-0/data/info.pb
08:04:22.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.218.126
08:04:22.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45411
08:04:22.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:04:22.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:04:22.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:04:22.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:04:22.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:04:22.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/master-0/logs
08:04:22.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:04:22.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:04:22.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:04:22.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:04:22.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:04:22.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 8537ed5bb5e0e6ac1e481ff961e6b9d08cce7671
08:04:22.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:04:22.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 30 Apr 2026 07:43:22 UTC on e7f111948823
08:04:22.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11675
08:04:22.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:04:22.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:22.711239 25309 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:04:22.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:22.712705 25309 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:04:22.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:22.775223 25338 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:04:22.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 30 08:04:22 dist-test-slave-h4q8 krb5kdc[24437](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777536262, etypes {rep=17 tkt=17 ses=17}, kudu/127.23.218.65@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:04:22.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:22.852591 24785 init.cc:295] Successfully reacquired a new kerberos TGT
08:04:23.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:23.061141 24904 heartbeater.cc:344] Connected to a master server at 127.23.218.125:44351
08:04:23.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:23.068085 25272 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "b5d001440eba49d5b0265effe47d3c2c" instance_seqno: 1777536253728401) as {username='kudu', principal='kudu/127.23.218.65@KRBTEST.COM'} at 127.23.218.65:35383; Asking this server to re-register.
08:04:23.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:23.070026 24904 heartbeater.cc:461] Registering TS with master...
08:04:23.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:23.074412 25272 ts_manager.cc:194] Registered new tserver with Master: b5d001440eba49d5b0265effe47d3c2c (127.23.218.65:33999)
08:04:23.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:23.232904 25272 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "eca5d45e7a234d32aceb91dd556eb91f" instance_seqno: 1777536258952071) as {username='kudu', principal='kudu/127.23.218.67@KRBTEST.COM'} at 127.23.218.67:60205; Asking this server to re-register.
08:04:23.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:23.235100 25202 heartbeater.cc:461] Registering TS with master...
08:04:23.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:23.238312 25272 ts_manager.cc:194] Registered new tserver with Master: eca5d45e7a234d32aceb91dd556eb91f (127.23.218.67:35221)
08:04:23.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:23.251962 25272 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "ce738aaf37fe46cab22a838800c45dbf" instance_seqno: 1777536256012901) as {username='kudu', principal='kudu/127.23.218.66@KRBTEST.COM'} at 127.23.218.66:56497; Asking this server to re-register.
08:04:23.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:23.253892 25051 heartbeater.cc:461] Registering TS with master...
08:04:23.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:23.257375 25272 ts_manager.cc:194] Registered new tserver with Master: ce738aaf37fe46cab22a838800c45dbf (127.23.218.66:34745)
08:04:23.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:23.507606 25355 raft_consensus.cc:493] T 00000000000000000000000000000000 P e1b4e8ad29a5471292df64bfbbcf85e3 [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:04:23.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:23.510332 25355 raft_consensus.cc:515] T 00000000000000000000000000000000 P e1b4e8ad29a5471292df64bfbbcf85e3 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "85f6c126e6444b7bae846a0c871b0e18" member_type: VOTER last_known_addr { host: "127.23.218.126" port: 42861 } } peers { permanent_uuid: "e1b4e8ad29a5471292df64bfbbcf85e3" member_type: VOTER last_known_addr { host: "127.23.218.125" port: 44351 } } peers { permanent_uuid: "24998cc13d3f4ba594cef4e87999214f" member_type: VOTER last_known_addr { host: "127.23.218.124" port: 32793 } }
08:04:23.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:23.527617 25256 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.23.218.126:42861: connect: Connection refused (error 111)
08:04:23.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:23.538574 25355 leader_election.cc:290] T 00000000000000000000000000000000 P e1b4e8ad29a5471292df64bfbbcf85e3 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 85f6c126e6444b7bae846a0c871b0e18 (127.23.218.126:42861), 24998cc13d3f4ba594cef4e87999214f (127.23.218.124:32793)
08:04:23.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:23.543901 25256 leader_election.cc:336] T 00000000000000000000000000000000 P e1b4e8ad29a5471292df64bfbbcf85e3 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 85f6c126e6444b7bae846a0c871b0e18 (127.23.218.126:42861): Network error: Client connection negotiation failed: client connection to 127.23.218.126:42861: connect: Connection refused (error 111)
08:04:23.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:23.550444 25255 leader_election.cc:336] T 00000000000000000000000000000000 P e1b4e8ad29a5471292df64bfbbcf85e3 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 24998cc13d3f4ba594cef4e87999214f (127.23.218.124:32793): Network error: Client connection negotiation failed: client connection to 127.23.218.124:32793: connect: Connection refused (error 111)
08:04:23.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:23.551255 25255 leader_election.cc:304] T 00000000000000000000000000000000 P e1b4e8ad29a5471292df64bfbbcf85e3 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: e1b4e8ad29a5471292df64bfbbcf85e3; no voters: 24998cc13d3f4ba594cef4e87999214f, 85f6c126e6444b7bae846a0c871b0e18
08:04:23.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:23.552716 25355 raft_consensus.cc:2749] T 00000000000000000000000000000000 P e1b4e8ad29a5471292df64bfbbcf85e3 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
08:04:24.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:22.778265 25339 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:04:24.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:24.160496 25337 debug-util.cc:398] Leaking SignalData structure 0x7b0800028d00 after lost signal to thread 25309
08:04:24.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:24.461954 25309 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 1.685s user 0.487s sys 0.737s
08:04:24.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:24.463027 25309 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 1.687s user 0.487s sys 0.737s
08:04:24.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:24.474244 25367 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:04:24.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:24.474313 25309 server_base.cc:1061] running on GCE node
08:04:24.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:24.476104 25309 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:04:24.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:24.479673 25309 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:04:24.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:24.482121 25309 hybrid_clock.cc:648] HybridClock initialized: now 1777536264482070 us; error 49 us; skew 500 ppm
08:04:24.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:24.483616 25309 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:04:24.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 30 08:04:24 dist-test-slave-h4q8 krb5kdc[24437](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777536264, etypes {rep=17 tkt=17 ses=17}, kudu/127.23.218.126@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:04:24.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:24.493664 25309 init.cc:377] Logged in from keytab as kudu/127.23.218.126@KRBTEST.COM (short username kudu)
08:04:24.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:24.531018 25309 webserver.cc:492] Webserver started at http://127.23.218.126:45411/ using document root <none> and password file <none>
08:04:24.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:24.532548 25309 fs_manager.cc:362] Metadata directory not provided
08:04:24.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:24.532991 25309 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:04:24.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:24.546103 25309 fs_manager.cc:714] Time spent opening directory manager: real 0.008s user 0.006s sys 0.001s
08:04:24.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:24.552879 25373 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:04:24.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:24.554507 25309 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.005s sys 0.001s
08:04:24.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:24.555009 25309 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/master-0/data,/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/master-0/wal
08:04:24.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "85f6c126e6444b7bae846a0c871b0e18"
08:04:24.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-04-30 08:04:03 on dist-test-slave-h4q8"
08:04:24.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:24.558410 25309 fs_report.cc:389] FS layout report
08:04:24.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:04:24.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/master-0/wal
08:04:24.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/master-0/wal
08:04:24.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/master-0/data/data
08:04:24.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:04:24.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:04:24.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:04:24.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:04:24.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:04:24.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:04:24.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:04:24.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:04:24.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:04:24.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:04:24.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:04:24.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:24.625128 25309 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:04:24.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:24.635293 25309 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:04:24.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:24.635828 25309 kserver.cc:163] Server-wide thread pool size limit: 3276
08:04:24.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:24.772308 25309 rpc_server.cc:307] RPC server started. Bound to: 127.23.218.126:42861
08:04:24.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:24.772375 25424 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.23.218.126:42861 every 8 connection(s)
08:04:24.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:24.776834 25309 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/master-0/data/info.pb
08:04:24.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:24.781672 24425 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskf7pa4T/build/tsan/bin/kudu as pid 25309
08:04:24.783 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:04:24.783 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.23.218.124:32793
08:04:24.784 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
08:04:24.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:24.789287 24425 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskf7pa4T/build/tsan/bin/kudu
08:04:24.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskf7pa4T/build/tsan/bin/kudu
08:04:24.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/master-2/wal
08:04:24.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/master-2/data
08:04:24.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:04:24.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:04:24.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:04:24.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:04:24.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:04:24.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:04:24.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/master-2/logs
08:04:24.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/master-2/data/info.pb
08:04:24.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:04:24.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:04:24.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:04:24.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:04:24.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:04:24.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:04:24.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:04:24.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:04:24.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:04:24.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:04:24.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:04:24.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:04:24.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.218.124:32793
08:04:24.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.218.124
08:04:24.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=40583
08:04:24.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/krb5kdc/kudu.keytab
08:04:24.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.23.218.124
08:04:24.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:04:24.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
08:04:24.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
08:04:24.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
08:04:24.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.218.84:37547
08:04:24.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:04:24.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:04:24.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:04:24.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:04:24.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.218.126:42861,127.23.218.125:44351,127.23.218.124:32793
08:04:24.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:04:24.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43537/jwks.json
08:04:24.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
08:04:24.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
08:04:24.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:24.815743 25425 sys_catalog.cc:263] Verifying existing consensus state
08:04:24.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:24.835600 25425 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 85f6c126e6444b7bae846a0c871b0e18: Bootstrap starting.
08:04:24.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:24.895958 25425 log.cc:826] T 00000000000000000000000000000000 P 85f6c126e6444b7bae846a0c871b0e18: Log is configured to *not* fsync() on all Append() calls
08:04:24.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:24.931569 25425 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 85f6c126e6444b7bae846a0c871b0e18: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
08:04:24.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:24.932721 25425 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 85f6c126e6444b7bae846a0c871b0e18: Bootstrap complete.
08:04:24.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:24.974923 25425 raft_consensus.cc:359] T 00000000000000000000000000000000 P 85f6c126e6444b7bae846a0c871b0e18 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "85f6c126e6444b7bae846a0c871b0e18" member_type: VOTER last_known_addr { host: "127.23.218.126" port: 42861 } } peers { permanent_uuid: "e1b4e8ad29a5471292df64bfbbcf85e3" member_type: VOTER last_known_addr { host: "127.23.218.125" port: 44351 } } peers { permanent_uuid: "24998cc13d3f4ba594cef4e87999214f" member_type: VOTER last_known_addr { host: "127.23.218.124" port: 32793 } }
08:04:24.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:24.978797 25425 raft_consensus.cc:740] T 00000000000000000000000000000000 P 85f6c126e6444b7bae846a0c871b0e18 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 85f6c126e6444b7bae846a0c871b0e18, State: Initialized, Role: FOLLOWER
08:04:24.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:24.980042 25425 consensus_queue.cc:260] T 00000000000000000000000000000000 P 85f6c126e6444b7bae846a0c871b0e18 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "85f6c126e6444b7bae846a0c871b0e18" member_type: VOTER last_known_addr { host: "127.23.218.126" port: 42861 } } peers { permanent_uuid: "e1b4e8ad29a5471292df64bfbbcf85e3" member_type: VOTER last_known_addr { host: "127.23.218.125" port: 44351 } } peers { permanent_uuid: "24998cc13d3f4ba594cef4e87999214f" member_type: VOTER last_known_addr { host: "127.23.218.124" port: 32793 } }
08:04:24.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:24.990167 25429 sys_catalog.cc:455] T 00000000000000000000000000000000 P 85f6c126e6444b7bae846a0c871b0e18 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "85f6c126e6444b7bae846a0c871b0e18" member_type: VOTER last_known_addr { host: "127.23.218.126" port: 42861 } } peers { permanent_uuid: "e1b4e8ad29a5471292df64bfbbcf85e3" member_type: VOTER last_known_addr { host: "127.23.218.125" port: 44351 } } peers { permanent_uuid: "24998cc13d3f4ba594cef4e87999214f" member_type: VOTER last_known_addr { host: "127.23.218.124" port: 32793 } } }
08:04:24.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:24.991070 25429 sys_catalog.cc:458] T 00000000000000000000000000000000 P 85f6c126e6444b7bae846a0c871b0e18 [sys.catalog]: This master's current role is: FOLLOWER
08:04:24.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:24.994931 25425 sys_catalog.cc:565] T 00000000000000000000000000000000 P 85f6c126e6444b7bae846a0c871b0e18 [sys.catalog]: configured and running, proceeding with master startup.
08:04:25.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:25.165946 25442 raft_consensus.cc:493] T 00000000000000000000000000000000 P e1b4e8ad29a5471292df64bfbbcf85e3 [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:04:25.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:25.166478 25442 raft_consensus.cc:515] T 00000000000000000000000000000000 P e1b4e8ad29a5471292df64bfbbcf85e3 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "85f6c126e6444b7bae846a0c871b0e18" member_type: VOTER last_known_addr { host: "127.23.218.126" port: 42861 } } peers { permanent_uuid: "e1b4e8ad29a5471292df64bfbbcf85e3" member_type: VOTER last_known_addr { host: "127.23.218.125" port: 44351 } } peers { permanent_uuid: "24998cc13d3f4ba594cef4e87999214f" member_type: VOTER last_known_addr { host: "127.23.218.124" port: 32793 } }
08:04:25.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 30 08:04:25 dist-test-slave-h4q8 krb5kdc[24437](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777536262, etypes {rep=17 tkt=17 ses=17}, kudu/127.23.218.65@KRBTEST.COM for kudu/127.23.218.126@KRBTEST.COM
08:04:25.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:25.195098 25442 leader_election.cc:290] T 00000000000000000000000000000000 P e1b4e8ad29a5471292df64bfbbcf85e3 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 85f6c126e6444b7bae846a0c871b0e18 (127.23.218.126:42861), 24998cc13d3f4ba594cef4e87999214f (127.23.218.124:32793)
08:04:25.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:25.218621 25255 leader_election.cc:336] T 00000000000000000000000000000000 P e1b4e8ad29a5471292df64bfbbcf85e3 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 24998cc13d3f4ba594cef4e87999214f (127.23.218.124:32793): Network error: Client connection negotiation failed: client connection to 127.23.218.124:32793: connect: Connection refused (error 111)
08:04:25.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 30 08:04:25 dist-test-slave-h4q8 krb5kdc[24437](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777536261, etypes {rep=17 tkt=17 ses=17}, kudu/127.23.218.125@KRBTEST.COM for kudu/127.23.218.126@KRBTEST.COM
08:04:25.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:25.260076 24906 heartbeater.cc:344] Connected to a master server at 127.23.218.126:42861
08:04:25.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:25.262856 25449 catalog_manager.cc:805] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
08:04:25.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:25.311281 25400 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "e1b4e8ad29a5471292df64bfbbcf85e3" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "85f6c126e6444b7bae846a0c871b0e18" is_pre_election: true
08:04:25.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:25.312330 25400 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 85f6c126e6444b7bae846a0c871b0e18 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate e1b4e8ad29a5471292df64bfbbcf85e3 in term 1.
08:04:25.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:25.313855 25256 leader_election.cc:304] T 00000000000000000000000000000000 P e1b4e8ad29a5471292df64bfbbcf85e3 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 85f6c126e6444b7bae846a0c871b0e18, e1b4e8ad29a5471292df64bfbbcf85e3; no voters: 24998cc13d3f4ba594cef4e87999214f
08:04:25.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:25.314898 25442 raft_consensus.cc:2804] T 00000000000000000000000000000000 P e1b4e8ad29a5471292df64bfbbcf85e3 [term 1 FOLLOWER]: Leader pre-election won for term 2
08:04:25.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:25.315222 25442 raft_consensus.cc:493] T 00000000000000000000000000000000 P e1b4e8ad29a5471292df64bfbbcf85e3 [term 1 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
08:04:25.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:25.315510 25442 raft_consensus.cc:3060] T 00000000000000000000000000000000 P e1b4e8ad29a5471292df64bfbbcf85e3 [term 1 FOLLOWER]: Advancing to term 2
08:04:25.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:25.327857 25442 raft_consensus.cc:515] T 00000000000000000000000000000000 P e1b4e8ad29a5471292df64bfbbcf85e3 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "85f6c126e6444b7bae846a0c871b0e18" member_type: VOTER last_known_addr { host: "127.23.218.126" port: 42861 } } peers { permanent_uuid: "e1b4e8ad29a5471292df64bfbbcf85e3" member_type: VOTER last_known_addr { host: "127.23.218.125" port: 44351 } } peers { permanent_uuid: "24998cc13d3f4ba594cef4e87999214f" member_type: VOTER last_known_addr { host: "127.23.218.124" port: 32793 } }
08:04:25.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:25.330899 25400 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "e1b4e8ad29a5471292df64bfbbcf85e3" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "85f6c126e6444b7bae846a0c871b0e18"
08:04:25.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:25.331506 25400 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 85f6c126e6444b7bae846a0c871b0e18 [term 1 FOLLOWER]: Advancing to term 2
08:04:25.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:25.336700 25255 leader_election.cc:336] T 00000000000000000000000000000000 P e1b4e8ad29a5471292df64bfbbcf85e3 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 24998cc13d3f4ba594cef4e87999214f (127.23.218.124:32793): Network error: Client connection negotiation failed: client connection to 127.23.218.124:32793: connect: Connection refused (error 111)
08:04:25.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:25.337616 25442 leader_election.cc:290] T 00000000000000000000000000000000 P e1b4e8ad29a5471292df64bfbbcf85e3 [CANDIDATE]: Term 2 election: Requested vote from peers 85f6c126e6444b7bae846a0c871b0e18 (127.23.218.126:42861), 24998cc13d3f4ba594cef4e87999214f (127.23.218.124:32793)
08:04:25.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:25.342605 25456 negotiation.cc:336] Failed RPC negotiation. Trace:
08:04:25.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0430 08:04:25.297389 (+ 0us) reactor.cc:678] Submitting negotiation task for server connection from 127.23.218.66:48249 (local address 127.23.218.126:42861)
08:04:25.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0430 08:04:25.300616 (+ 3227us) server_negotiation.cc:207] Beginning negotiation
08:04:25.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0430 08:04:25.300642 (+ 26us) server_negotiation.cc:400] Waiting for connection header
08:04:25.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0430 08:04:25.329321 (+ 28679us) server_negotiation.cc:408] Connection header received
08:04:25.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0430 08:04:25.329481 (+ 160us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
08:04:25.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0430 08:04:25.329514 (+ 33us) server_negotiation.cc:462] Received NEGOTIATE request from client
08:04:25.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0430 08:04:25.329961 (+ 447us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
08:04:25.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0430 08:04:25.330152 (+ 191us) server_negotiation.cc:227] Negotiated authn=SASL
08:04:25.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0430 08:04:25.336235 (+ 6083us) negotiation.cc:326] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.23.218.66:48249: BlockingRecv error: recv got EOF from 127.23.218.66:48249 (error 108)
08:04:25.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":2625,"thread_start_us":2408,"threads_started":1}
08:04:25.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:25.347083 25400 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 85f6c126e6444b7bae846a0c871b0e18 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e1b4e8ad29a5471292df64bfbbcf85e3 in term 2.
08:04:25.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:25.348215 25256 leader_election.cc:304] T 00000000000000000000000000000000 P e1b4e8ad29a5471292df64bfbbcf85e3 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 85f6c126e6444b7bae846a0c871b0e18, e1b4e8ad29a5471292df64bfbbcf85e3; no voters: 24998cc13d3f4ba594cef4e87999214f
08:04:25.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:25.349512 25442 raft_consensus.cc:2804] T 00000000000000000000000000000000 P e1b4e8ad29a5471292df64bfbbcf85e3 [term 2 FOLLOWER]: Leader election won for term 2
08:04:25.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 30 08:04:25 dist-test-slave-h4q8 krb5kdc[24437](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777536264, etypes {rep=17 tkt=17 ses=17}, kudu/127.23.218.126@KRBTEST.COM for kudu/127.23.218.126@KRBTEST.COM
08:04:25.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:25.358350 25442 raft_consensus.cc:697] T 00000000000000000000000000000000 P e1b4e8ad29a5471292df64bfbbcf85e3 [term 2 LEADER]: Becoming Leader. State: Replica: e1b4e8ad29a5471292df64bfbbcf85e3, State: Running, Role: LEADER
08:04:25.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:25.359724 25442 consensus_queue.cc:237] T 00000000000000000000000000000000 P e1b4e8ad29a5471292df64bfbbcf85e3 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "85f6c126e6444b7bae846a0c871b0e18" member_type: VOTER last_known_addr { host: "127.23.218.126" port: 42861 } } peers { permanent_uuid: "e1b4e8ad29a5471292df64bfbbcf85e3" member_type: VOTER last_known_addr { host: "127.23.218.125" port: 44351 } } peers { permanent_uuid: "24998cc13d3f4ba594cef4e87999214f" member_type: VOTER last_known_addr { host: "127.23.218.124" port: 32793 } }
08:04:25.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:25.365474 25206 heartbeater.cc:344] Connected to a master server at 127.23.218.126:42861
08:04:25.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:25.371465 25390 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "eca5d45e7a234d32aceb91dd556eb91f" instance_seqno: 1777536258952071) as {username='kudu', principal='kudu/127.23.218.67@KRBTEST.COM'} at 127.23.218.67:35947; Asking this server to re-register.
08:04:25.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:25.373898 25206 heartbeater.cc:461] Registering TS with master...
08:04:25.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:25.378042 25390 ts_manager.cc:194] Registered new tserver with Master: eca5d45e7a234d32aceb91dd556eb91f (127.23.218.67:35221)
08:04:25.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:25.384797 25457 sys_catalog.cc:455] T 00000000000000000000000000000000 P e1b4e8ad29a5471292df64bfbbcf85e3 [sys.catalog]: SysCatalogTable state changed. Reason: New leader e1b4e8ad29a5471292df64bfbbcf85e3. Latest consensus state: current_term: 2 leader_uuid: "e1b4e8ad29a5471292df64bfbbcf85e3" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "85f6c126e6444b7bae846a0c871b0e18" member_type: VOTER last_known_addr { host: "127.23.218.126" port: 42861 } } peers { permanent_uuid: "e1b4e8ad29a5471292df64bfbbcf85e3" member_type: VOTER last_known_addr { host: "127.23.218.125" port: 44351 } } peers { permanent_uuid: "24998cc13d3f4ba594cef4e87999214f" member_type: VOTER last_known_addr { host: "127.23.218.124" port: 32793 } } }
08:04:25.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:25.385702 25457 sys_catalog.cc:458] T 00000000000000000000000000000000 P e1b4e8ad29a5471292df64bfbbcf85e3 [sys.catalog]: This master's current role is: LEADER
08:04:25.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:25.398891 25459 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
08:04:25.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:25.408344 25459 catalog_manager.cc:1494] Initializing Kudu cluster ID...
08:04:25.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 30 08:04:25 dist-test-slave-h4q8 krb5kdc[24437](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777536264, etypes {rep=17 tkt=17 ses=17}, kudu/127.23.218.126@KRBTEST.COM for kudu/127.23.218.125@KRBTEST.COM
08:04:25.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:25.424919 25459 catalog_manager.cc:1269] Loaded cluster ID: f8b4d76ecadd440bbbfe796430e93114
08:04:25.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:25.425225 25459 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
08:04:25.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:25.430611 25459 catalog_manager.cc:1514] Loading token signing keys...
08:04:25.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:25.435364 25459 catalog_manager.cc:6055] T 00000000000000000000000000000000 P e1b4e8ad29a5471292df64bfbbcf85e3: Loaded TSK: 0
08:04:25.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:25.436597 25459 catalog_manager.cc:1524] Initializing in-progress tserver states...
08:04:25.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:25.443610 25309 master_runner.cc:186] Error getting master registration for 127.23.218.124:32793: Network error: Client connection negotiation failed: client connection to 127.23.218.124:32793: connect: Connection refused (error 111),
08:04:25.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:25.468798 25309 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
08:04:25.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:25.724803 25427 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
08:04:25.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:25.725687 25427 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:04:25.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:25.727031 25427 flags.cc:432] Enabled unsafe flag: --never_fsync=true
08:04:25.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:25.781178 25400 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 85f6c126e6444b7bae846a0c871b0e18 [term 2 FOLLOWER]: Refusing update from remote peer e1b4e8ad29a5471292df64bfbbcf85e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 2 index: 5. (index mismatch)
08:04:25.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 30 08:04:25 dist-test-slave-h4q8 krb5kdc[24437](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777536265, etypes {rep=17 tkt=17 ses=17}, kudu/127.23.218.66@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:04:25.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:25.783099 25457 consensus_queue.cc:1048] T 00000000000000000000000000000000 P e1b4e8ad29a5471292df64bfbbcf85e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "85f6c126e6444b7bae846a0c871b0e18" member_type: VOTER last_known_addr { host: "127.23.218.126" port: 42861 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
08:04:25.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:25.783947 24933 init.cc:295] Successfully reacquired a new kerberos TGT
08:04:25.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:25.802160 25427 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
08:04:25.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:25.802557 25427 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
08:04:25.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:25.803092 25427 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
08:04:25.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:25.803489 25427 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
08:04:25.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:25.803864 25427 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
08:04:25.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:25.804241 25427 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
08:04:25.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:25.804525 25427 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:43537/jwks.json
08:04:25.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:25.815183 25467 sys_catalog.cc:455] T 00000000000000000000000000000000 P 85f6c126e6444b7bae846a0c871b0e18 [sys.catalog]: SysCatalogTable state changed. Reason: New leader e1b4e8ad29a5471292df64bfbbcf85e3. Latest consensus state: current_term: 2 leader_uuid: "e1b4e8ad29a5471292df64bfbbcf85e3" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "85f6c126e6444b7bae846a0c871b0e18" member_type: VOTER last_known_addr { host: "127.23.218.126" port: 42861 } } peers { permanent_uuid: "e1b4e8ad29a5471292df64bfbbcf85e3" member_type: VOTER last_known_addr { host: "127.23.218.125" port: 44351 } } peers { permanent_uuid: "24998cc13d3f4ba594cef4e87999214f" member_type: VOTER last_known_addr { host: "127.23.218.124" port: 32793 } } }
08:04:25.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:25.815877 25467 sys_catalog.cc:458] T 00000000000000000000000000000000 P 85f6c126e6444b7bae846a0c871b0e18 [sys.catalog]: This master's current role is: FOLLOWER
08:04:25.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:25.851181 25442 sys_catalog.cc:455] T 00000000000000000000000000000000 P e1b4e8ad29a5471292df64bfbbcf85e3 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "e1b4e8ad29a5471292df64bfbbcf85e3" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "85f6c126e6444b7bae846a0c871b0e18" member_type: VOTER last_known_addr { host: "127.23.218.126" port: 42861 } } peers { permanent_uuid: "e1b4e8ad29a5471292df64bfbbcf85e3" member_type: VOTER last_known_addr { host: "127.23.218.125" port: 44351 } } peers { permanent_uuid: "24998cc13d3f4ba594cef4e87999214f" member_type: VOTER last_known_addr { host: "127.23.218.124" port: 32793 } } }
08:04:25.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:25.851908 25442 sys_catalog.cc:458] T 00000000000000000000000000000000 P e1b4e8ad29a5471292df64bfbbcf85e3 [sys.catalog]: This master's current role is: LEADER
08:04:25.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:25.848183 25467 sys_catalog.cc:455] T 00000000000000000000000000000000 P 85f6c126e6444b7bae846a0c871b0e18 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "e1b4e8ad29a5471292df64bfbbcf85e3" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "85f6c126e6444b7bae846a0c871b0e18" member_type: VOTER last_known_addr { host: "127.23.218.126" port: 42861 } } peers { permanent_uuid: "e1b4e8ad29a5471292df64bfbbcf85e3" member_type: VOTER last_known_addr { host: "127.23.218.125" port: 44351 } } peers { permanent_uuid: "24998cc13d3f4ba594cef4e87999214f" member_type: VOTER last_known_addr { host: "127.23.218.124" port: 32793 } } }
08:04:25.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:25.852982 25467 sys_catalog.cc:458] T 00000000000000000000000000000000 P 85f6c126e6444b7bae846a0c871b0e18 [sys.catalog]: This master's current role is: FOLLOWER
08:04:25.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:25.868920 25255 consensus_peers.cc:597] T 00000000000000000000000000000000 P e1b4e8ad29a5471292df64bfbbcf85e3 -> Peer 24998cc13d3f4ba594cef4e87999214f (127.23.218.124:32793): Couldn't send request to peer 24998cc13d3f4ba594cef4e87999214f. Status: Network error: Client connection negotiation failed: client connection to 127.23.218.124:32793: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
08:04:25.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:25.897308 25427 master_runner.cc:386] Master server non-default flags:
08:04:25.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:04:25.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.218.84:37547
08:04:25.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:04:25.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:04:25.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/master-2/data
08:04:25.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/master-2/wal
08:04:25.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
08:04:25.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:04:25.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.218.126:42861,127.23.218.125:44351,127.23.218.124:32793
08:04:25.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/krb5kdc/kudu.keytab
08:04:25.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.23.218.124
08:04:25.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:04:25.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:04:25.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:04:25.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.218.124:32793
08:04:25.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:04:25.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:04:25.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:04:25.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43537/jwks.json
08:04:25.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:04:25.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
08:04:25.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
08:04:25.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:04:25.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:04:25.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/master-2/data/info.pb
08:04:25.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.218.124
08:04:25.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=40583
08:04:25.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:04:25.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:04:25.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:04:25.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:04:25.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:04:25.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/master-2/logs
08:04:25.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:04:25.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:04:25.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:04:25.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:04:25.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:04:25.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 8537ed5bb5e0e6ac1e481ff961e6b9d08cce7671
08:04:25.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:04:25.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 30 Apr 2026 07:43:22 UTC on e7f111948823
08:04:25.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11675
08:04:25.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:04:25.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:25.899121 25427 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:04:25.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:25.900514 25427 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:04:25.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:25.914518 25475 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:04:26.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:26.273451 25390 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "b5d001440eba49d5b0265effe47d3c2c" instance_seqno: 1777536253728401) as {username='kudu', principal='kudu/127.23.218.65@KRBTEST.COM'} at 127.23.218.65:38231; Asking this server to re-register.
08:04:26.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:26.274518 25449 catalog_manager.cc:1269] Loaded cluster ID: f8b4d76ecadd440bbbfe796430e93114
08:04:26.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:26.277910 25449 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 85f6c126e6444b7bae846a0c871b0e18: loading cluster ID for follower catalog manager: success
08:04:26.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:26.278141 24906 heartbeater.cc:461] Registering TS with master...
08:04:26.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:26.281771 25390 ts_manager.cc:194] Registered new tserver with Master: b5d001440eba49d5b0265effe47d3c2c (127.23.218.65:33999)
08:04:26.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:26.291500 25449 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 85f6c126e6444b7bae846a0c871b0e18: acquiring CA information for follower catalog manager: success
08:04:26.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:26.299393 25449 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 85f6c126e6444b7bae846a0c871b0e18: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
08:04:26.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:26.363962 25054 heartbeater.cc:344] Connected to a master server at 127.23.218.126:42861
08:04:26.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:26.367120 25390 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "ce738aaf37fe46cab22a838800c45dbf" instance_seqno: 1777536256012901) as {username='kudu', principal='kudu/127.23.218.66@KRBTEST.COM'} at 127.23.218.66:36565; Asking this server to re-register.
08:04:26.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:26.371665 25054 heartbeater.cc:461] Registering TS with master...
08:04:26.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:26.375401 25390 ts_manager.cc:194] Registered new tserver with Master: ce738aaf37fe46cab22a838800c45dbf (127.23.218.66:34745)
08:04:27.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:27.088894 24904 heartbeater.cc:499] Master 127.23.218.125:44351 was elected leader, sending a full tablet report...
08:04:27.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:27.253077 25202 heartbeater.cc:499] Master 127.23.218.125:44351 was elected leader, sending a full tablet report...
08:04:27.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:27.276942 25051 heartbeater.cc:499] Master 127.23.218.125:44351 was elected leader, sending a full tablet report...
08:04:27.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:27.398134 25218 master_runner.cc:186] Error getting master registration for 127.23.218.124:32793: Network error: Client connection negotiation failed: client connection to 127.23.218.124:32793: connect: Connection refused (error 111),
08:04:27.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:27.398612 25218 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
08:04:27.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:27.325065 25474 debug-util.cc:398] Leaking SignalData structure 0x7b0800028d00 after lost signal to thread 25427
08:04:27.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:25.915617 25476 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:04:27.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:27.767490 25427 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 1.853s user 0.617s sys 1.103s
08:04:27.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:27.767833 25427 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 1.854s user 0.617s sys 1.103s
08:04:27.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:27.771539 25421 debug-util.cc:398] Leaking SignalData structure 0x7b08000714c0 after lost signal to thread 25332
08:04:27.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:27.772258 25421 debug-util.cc:398] Leaking SignalData structure 0x7b0800071580 after lost signal to thread 25424
08:04:27.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:27.780678 25496 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:04:27.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:27.781977 25427 server_base.cc:1061] running on GCE node
08:04:27.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:27.783529 25427 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:04:27.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:27.799572 25427 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:04:27.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:27.801406 25427 hybrid_clock.cc:648] HybridClock initialized: now 1777536267801365 us; error 58 us; skew 500 ppm
08:04:27.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:27.802587 25427 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:04:27.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 30 08:04:27 dist-test-slave-h4q8 krb5kdc[24437](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777536267, etypes {rep=17 tkt=17 ses=17}, kudu/127.23.218.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:04:27.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:27.815500 25427 init.cc:377] Logged in from keytab as kudu/127.23.218.124@KRBTEST.COM (short username kudu)
08:04:27.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:27.839555 25427 webserver.cc:492] Webserver started at http://127.23.218.124:40583/ using document root <none> and password file <none>
08:04:27.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:27.840976 25427 fs_manager.cc:362] Metadata directory not provided
08:04:27.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:27.841524 25427 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:04:27.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:27.857618 25427 fs_manager.cc:714] Time spent opening directory manager: real 0.006s user 0.005s sys 0.001s
08:04:27.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:27.862959 25503 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:04:27.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:27.864279 25427 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.001s sys 0.002s
08:04:27.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:27.864642 25427 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/master-2/data,/tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/master-2/wal
08:04:27.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "24998cc13d3f4ba594cef4e87999214f"
08:04:27.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-04-30 08:04:08 on dist-test-slave-h4q8"
08:04:27.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:27.886245 25427 fs_report.cc:389] FS layout report
08:04:27.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:04:27.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/master-2/wal
08:04:27.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/master-2/wal
08:04:27.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/master-2/data/data
08:04:27.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:04:27.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:04:27.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:04:27.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:04:27.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:04:27.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:04:27.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:04:27.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:04:27.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:04:27.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:04:27.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:04:27.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:27.967373 25427 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:04:27.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:27.976610 25427 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:04:27.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:27.977483 25427 kserver.cc:163] Server-wide thread pool size limit: 3276
08:04:28.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:28.243935 25427 rpc_server.cc:307] RPC server started. Bound to: 127.23.218.124:32793
08:04:28.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:28.243994 25557 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.23.218.124:32793 every 8 connection(s)
08:04:28.250 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:04:28.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:28.247104 25427 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskf7pa4T/test-tmp/mini-kudu-cluster13696900367268570772/master-2/data/info.pb
08:04:28.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:28.248814 24425 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskf7pa4T/build/tsan/bin/kudu as pid 25427
08:04:28.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:28.298413 25558 sys_catalog.cc:263] Verifying existing consensus state
08:04:28.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 30 08:04:28 dist-test-slave-h4q8 krb5kdc[24437](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777536261, etypes {rep=17 tkt=17 ses=17}, kudu/127.23.218.125@KRBTEST.COM for kudu/127.23.218.124@KRBTEST.COM
08:04:28.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:28.308513 25558 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 24998cc13d3f4ba594cef4e87999214f: Bootstrap starting.
08:04:28.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:28.372638 25561 negotiation.cc:336] Failed RPC negotiation. Trace:
08:04:28.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0430 08:04:28.342471 (+ 0us) reactor.cc:678] Submitting negotiation task for server connection from 127.23.218.67:55993 (local address 127.23.218.124:32793)
08:04:28.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0430 08:04:28.343552 (+ 1081us) server_negotiation.cc:207] Beginning negotiation
08:04:28.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0430 08:04:28.343591 (+ 39us) server_negotiation.cc:400] Waiting for connection header
08:04:28.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0430 08:04:28.367636 (+ 24045us) server_negotiation.cc:408] Connection header received
08:04:28.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0430 08:04:28.368187 (+ 551us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
08:04:28.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0430 08:04:28.368238 (+ 51us) server_negotiation.cc:462] Received NEGOTIATE request from client
08:04:28.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0430 08:04:28.368582 (+ 344us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
08:04:28.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0430 08:04:28.368776 (+ 194us) server_negotiation.cc:227] Negotiated authn=SASL
08:04:28.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0430 08:04:28.370941 (+ 2165us) negotiation.cc:326] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.23.218.67:55993: BlockingRecv error: recv got EOF from 127.23.218.67:55993 (error 108)
08:04:28.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":197}
08:04:28.395 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.23.218.125:44351(127.23.218.125:44351)] connecting to peer
08:04:28.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 30 08:04:28 dist-test-slave-h4q8 krb5kdc[24437](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777536265, etypes {rep=17 tkt=17 ses=17}, kudu/127.23.218.66@KRBTEST.COM for kudu/127.23.218.124@KRBTEST.COM
08:04:28.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:28.434988 25558 log.cc:826] T 00000000000000000000000000000000 P 24998cc13d3f4ba594cef4e87999214f: Log is configured to *not* fsync() on all Append() calls
08:04:28.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:28.454751 25052 heartbeater.cc:344] Connected to a master server at 127.23.218.124:32793
08:04:28.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 30 08:04:28 dist-test-slave-h4q8 krb5kdc[24437](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777536268, etypes {rep=17 tkt=17 ses=17}, kudu/127.23.218.67@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:04:28.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:28.490202 25083 init.cc:295] Successfully reacquired a new kerberos TGT
08:04:28.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:28.495972 25558 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 24998cc13d3f4ba594cef4e87999214f: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
08:04:28.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:28.497624 25558 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 24998cc13d3f4ba594cef4e87999214f: Bootstrap complete.
08:04:28.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:28.550503 25558 raft_consensus.cc:359] T 00000000000000000000000000000000 P 24998cc13d3f4ba594cef4e87999214f [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "85f6c126e6444b7bae846a0c871b0e18" member_type: VOTER last_known_addr { host: "127.23.218.126" port: 42861 } } peers { permanent_uuid: "e1b4e8ad29a5471292df64bfbbcf85e3" member_type: VOTER last_known_addr { host: "127.23.218.125" port: 44351 } } peers { permanent_uuid: "24998cc13d3f4ba594cef4e87999214f" member_type: VOTER last_known_addr { host: "127.23.218.124" port: 32793 } }
08:04:28.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:28.556367 25558 raft_consensus.cc:740] T 00000000000000000000000000000000 P 24998cc13d3f4ba594cef4e87999214f [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 24998cc13d3f4ba594cef4e87999214f, State: Initialized, Role: FOLLOWER
08:04:28.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:28.558123 25558 consensus_queue.cc:260] T 00000000000000000000000000000000 P 24998cc13d3f4ba594cef4e87999214f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "85f6c126e6444b7bae846a0c871b0e18" member_type: VOTER last_known_addr { host: "127.23.218.126" port: 42861 } } peers { permanent_uuid: "e1b4e8ad29a5471292df64bfbbcf85e3" member_type: VOTER last_known_addr { host: "127.23.218.125" port: 44351 } } peers { permanent_uuid: "24998cc13d3f4ba594cef4e87999214f" member_type: VOTER last_known_addr { host: "127.23.218.124" port: 32793 } }
08:04:28.572 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.23.218.126:42861(127.23.218.126:42861)] connecting to peer
08:04:28.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:28.574365 25532 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 24998cc13d3f4ba594cef4e87999214f [term 1 FOLLOWER]: Advancing to term 2
08:04:28.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:28.584998 25558 sys_catalog.cc:565] T 00000000000000000000000000000000 P 24998cc13d3f4ba594cef4e87999214f [sys.catalog]: configured and running, proceeding with master startup.
08:04:28.590 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.23.218.124:32793(127.23.218.124:32793)] connecting to peer
08:04:28.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:28.632328 25565 sys_catalog.cc:455] T 00000000000000000000000000000000 P 24998cc13d3f4ba594cef4e87999214f [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 2 leader_uuid: "e1b4e8ad29a5471292df64bfbbcf85e3" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "85f6c126e6444b7bae846a0c871b0e18" member_type: VOTER last_known_addr { host: "127.23.218.126" port: 42861 } } peers { permanent_uuid: "e1b4e8ad29a5471292df64bfbbcf85e3" member_type: VOTER last_known_addr { host: "127.23.218.125" port: 44351 } } peers { permanent_uuid: "24998cc13d3f4ba594cef4e87999214f" member_type: VOTER last_known_addr { host: "127.23.218.124" port: 32793 } } }
08:04:28.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:28.633232 25565 sys_catalog.cc:458] T 00000000000000000000000000000000 P 24998cc13d3f4ba594cef4e87999214f [sys.catalog]: This master's current role is: FOLLOWER
08:04:28.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:28.634150 25568 sys_catalog.cc:455] T 00000000000000000000000000000000 P 24998cc13d3f4ba594cef4e87999214f [sys.catalog]: SysCatalogTable state changed. Reason: New leader e1b4e8ad29a5471292df64bfbbcf85e3. Latest consensus state: current_term: 2 leader_uuid: "e1b4e8ad29a5471292df64bfbbcf85e3" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "85f6c126e6444b7bae846a0c871b0e18" member_type: VOTER last_known_addr { host: "127.23.218.126" port: 42861 } } peers { permanent_uuid: "e1b4e8ad29a5471292df64bfbbcf85e3" member_type: VOTER last_known_addr { host: "127.23.218.125" port: 44351 } } peers { permanent_uuid: "24998cc13d3f4ba594cef4e87999214f" member_type: VOTER last_known_addr { host: "127.23.218.124" port: 32793 } } }
08:04:28.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:28.634979 25568 sys_catalog.cc:458] T 00000000000000000000000000000000 P 24998cc13d3f4ba594cef4e87999214f [sys.catalog]: This master's current role is: FOLLOWER
08:04:28.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:28.637405 25575 sys_catalog.cc:455] T 00000000000000000000000000000000 P 24998cc13d3f4ba594cef4e87999214f [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "e1b4e8ad29a5471292df64bfbbcf85e3" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "85f6c126e6444b7bae846a0c871b0e18" member_type: VOTER last_known_addr { host: "127.23.218.126" port: 42861 } } peers { permanent_uuid: "e1b4e8ad29a5471292df64bfbbcf85e3" member_type: VOTER last_known_addr { host: "127.23.218.125" port: 44351 } } peers { permanent_uuid: "24998cc13d3f4ba594cef4e87999214f" member_type: VOTER last_known_addr { host: "127.23.218.124" port: 32793 } } }
08:04:28.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:28.639443 25575 sys_catalog.cc:458] T 00000000000000000000000000000000 P 24998cc13d3f4ba594cef4e87999214f [sys.catalog]: This master's current role is: FOLLOWER
08:04:28.658 [DEBUG - kudu-nio-2] (Connection.java:757) [peer master-127.23.218.124:32793(127.23.218.124:32793)] Successfully connected to peer
08:04:28.662 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.23.218.126:42861(127.23.218.126:42861)] Successfully connected to peer
08:04:28.675 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.23.218.125:44351(127.23.218.125:44351)] Successfully connected to peer
08:04:28.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 30 08:04:28 dist-test-slave-h4q8 krb5kdc[24437](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777536267, etypes {rep=17 tkt=17 ses=17}, kudu/127.23.218.124@KRBTEST.COM for kudu/127.23.218.126@KRBTEST.COM
08:04:28.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:28.870764 25585 catalog_manager.cc:1269] Loaded cluster ID: f8b4d76ecadd440bbbfe796430e93114
08:04:28.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:28.895890 25585 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 24998cc13d3f4ba594cef4e87999214f: loading cluster ID for follower catalog manager: success
08:04:28.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:28.902268 25491 sys_catalog.cc:455] T 00000000000000000000000000000000 P e1b4e8ad29a5471292df64bfbbcf85e3 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "e1b4e8ad29a5471292df64bfbbcf85e3" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "85f6c126e6444b7bae846a0c871b0e18" member_type: VOTER last_known_addr { host: "127.23.218.126" port: 42861 } } peers { permanent_uuid: "e1b4e8ad29a5471292df64bfbbcf85e3" member_type: VOTER last_known_addr { host: "127.23.218.125" port: 44351 } } peers { permanent_uuid: "24998cc13d3f4ba594cef4e87999214f" member_type: VOTER last_known_addr { host: "127.23.218.124" port: 32793 } } }
08:04:28.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:28.907434 25491 sys_catalog.cc:458] T 00000000000000000000000000000000 P e1b4e8ad29a5471292df64bfbbcf85e3 [sys.catalog]: This master's current role is: LEADER
08:04:28.908 [WARN - kudu-nio-0] (SecurityContext.java:206) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
08:04:28.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:28.915455 25585 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 24998cc13d3f4ba594cef4e87999214f: acquiring CA information for follower catalog manager: success
08:04:28.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:28.922132 25585 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 24998cc13d3f4ba594cef4e87999214f: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
08:04:28.969 [DEBUG - kudu-nio-2] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.23.218.124
08:04:28.970 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.23.218.125
08:04:28.970 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.23.218.126
08:04:28.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 30 08:04:28 dist-test-slave-h4q8 krb5kdc[24437](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777536267, etypes {rep=17 tkt=17 ses=17}, kudu/127.23.218.124@KRBTEST.COM for kudu/127.23.218.125@KRBTEST.COM
08:04:29.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:29.275674 24905 heartbeater.cc:344] Connected to a master server at 127.23.218.124:32793
08:04:29.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:29.282209 25522 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "b5d001440eba49d5b0265effe47d3c2c" instance_seqno: 1777536253728401) as {username='kudu', principal='kudu/127.23.218.65@KRBTEST.COM'} at 127.23.218.65:50725; Asking this server to re-register.
08:04:29.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:29.283627 24905 heartbeater.cc:461] Registering TS with master...
08:04:29.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:29.287245 25522 ts_manager.cc:194] Registered new tserver with Master: b5d001440eba49d5b0265effe47d3c2c (127.23.218.65:33999)
08:04:29.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:29.437477 25203 heartbeater.cc:344] Connected to a master server at 127.23.218.124:32793
08:04:29.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:29.438081 25203 heartbeater.cc:499] Master 127.23.218.124:32793 was elected leader, sending a full tablet report...
08:04:29.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:29.440831 25522 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "eca5d45e7a234d32aceb91dd556eb91f" instance_seqno: 1777536258952071) as {username='kudu', principal='kudu/127.23.218.67@KRBTEST.COM'} at 127.23.218.67:54395; Asking this server to re-register.
08:04:29.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:29.442245 25203 heartbeater.cc:461] Registering TS with master...
08:04:29.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:29.445408 25522 ts_manager.cc:194] Registered new tserver with Master: eca5d45e7a234d32aceb91dd556eb91f (127.23.218.67:35221)
08:04:29.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:29.460892 25522 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "ce738aaf37fe46cab22a838800c45dbf" instance_seqno: 1777536256012901) as {username='kudu', principal='kudu/127.23.218.66@KRBTEST.COM'} at 127.23.218.66:33885; Asking this server to re-register.
08:04:29.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:29.462373 25052 heartbeater.cc:461] Registering TS with master...
08:04:29.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:29.466114 25522 ts_manager.cc:194] Registered new tserver with Master: ce738aaf37fe46cab22a838800c45dbf (127.23.218.66:34745)
08:04:29.624 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.218.125:44351(127.23.218.125:44351)] cleaning up while in state NEGOTIATING due to: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
08:04:29.624 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.218.125:44351: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
08:04:29.627 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.218.126:42861(127.23.218.126:42861)] cleaning up while in state NEGOTIATING due to: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
08:04:29.628 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.218.126:42861: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
08:04:29.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:29.631603 25578 negotiation.cc:336] Failed RPC negotiation. Trace:
08:04:29.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0430 08:04:28.648396 (+ 0us) reactor.cc:678] Submitting negotiation task for server connection from 127.0.0.1:35128 (local address 127.23.218.126:42861)
08:04:29.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0430 08:04:28.709012 (+ 60616us) server_negotiation.cc:207] Beginning negotiation
08:04:29.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0430 08:04:28.709042 (+ 30us) server_negotiation.cc:400] Waiting for connection header
08:04:29.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0430 08:04:28.832632 (+123590us) server_negotiation.cc:408] Connection header received
08:04:29.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0430 08:04:28.872403 (+ 39771us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
08:04:29.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0430 08:04:28.872452 (+ 49us) server_negotiation.cc:462] Received NEGOTIATE request from client
08:04:29.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0430 08:04:28.872765 (+ 313us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
08:04:29.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0430 08:04:28.873020 (+ 255us) server_negotiation.cc:227] Negotiated authn=SASL
08:04:29.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0430 08:04:29.486717 (+613697us) server_negotiation.cc:366] Received TLS_HANDSHAKE NegotiatePB request
08:04:29.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0430 08:04:29.488207 (+ 1490us) server_negotiation.cc:378] Sending TLS_HANDSHAKE NegotiatePB response
08:04:29.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0430 08:04:29.606939 (+118732us) server_negotiation.cc:366] Received TLS_HANDSHAKE NegotiatePB request
08:04:29.634 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.23.218.126:42861(127.23.218.126:42861)] handling channelInactive
08:04:29.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0430 08:04:29.607523 (+ 584us) server_negotiation.cc:663] Negotiated TLSv1.3 with cipher TLS_AES_128_GCM_SHA256 TLSv1.3 Kx=any Au=any Enc=AESGCM(128) Mac=AEAD
08:04:29.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0430 08:04:29.630189 (+ 22666us) negotiation.cc:326] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:35128: BlockingRecv error: failed to read from TLS socket (remote: 127.0.0.1:35128): Cannot send after transport endpoint shutdown (error 108)
08:04:29.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":59918,"thread_start_us":57729,"threads_started":1}
08:04:29.634 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.23.218.125:44351(127.23.218.125:44351)] handling channelInactive
08:04:29.633 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.23.218.124:32793(127.23.218.124:32793)] cleaning up while in state NEGOTIATING due to: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
08:04:29.634 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.23.218.124:32793: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
08:04:29.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:29.636572 25581 negotiation.cc:336] Failed RPC negotiation. Trace:
08:04:29.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0430 08:04:28.665031 (+ 0us) reactor.cc:678] Submitting negotiation task for server connection from 127.0.0.1:50930 (local address 127.23.218.125:44351)
08:04:29.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0430 08:04:28.697679 (+ 32648us) server_negotiation.cc:207] Beginning negotiation
08:04:29.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0430 08:04:28.697718 (+ 39us) server_negotiation.cc:400] Waiting for connection header
08:04:29.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0430 08:04:28.827849 (+130131us) server_negotiation.cc:408] Connection header received
08:04:29.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0430 08:04:28.880910 (+ 53061us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
08:04:29.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0430 08:04:28.880955 (+ 45us) server_negotiation.cc:462] Received NEGOTIATE request from client
08:04:29.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0430 08:04:28.881227 (+ 272us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
08:04:29.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0430 08:04:28.881398 (+ 171us) server_negotiation.cc:227] Negotiated authn=SASL
08:04:29.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0430 08:04:29.481392 (+599994us) server_negotiation.cc:366] Received TLS_HANDSHAKE NegotiatePB request
08:04:29.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0430 08:04:29.482878 (+ 1486us) server_negotiation.cc:378] Sending TLS_HANDSHAKE NegotiatePB response
08:04:29.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0430 08:04:29.610735 (+127857us) server_negotiation.cc:366] Received TLS_HANDSHAKE NegotiatePB request
08:04:29.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0430 08:04:29.611400 (+ 665us) server_negotiation.cc:663] Negotiated TLSv1.3 with cipher TLS_AES_128_GCM_SHA256 TLSv1.3 Kx=any Au=any Enc=AESGCM(128) Mac=AEAD
08:04:29.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0430 08:04:29.634745 (+ 23345us) negotiation.cc:326] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:50930: BlockingRecv error: failed to read from TLS socket (remote: 127.0.0.1:50930): Cannot send after transport endpoint shutdown (error 108)
08:04:29.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":31768,"thread_start_us":31478,"threads_started":1}
08:04:29.638 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1523) Notify RPC KuduRpc(method=ListTabletServers, tablet=null, attempt=1, TimeoutTracker(timeout=30000, elapsed=1376), Traces: [0ms] refreshing cache from master, [72ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.218.125:44351, [287ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.218.126:42861, [306ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.218.124:32793, [1340ms] Sub RPC ConnectToMaster: received response from server master-127.23.218.125:44351: Configuration error: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), [1344ms] Sub RPC ConnectToMaster: received response from server master-127.23.218.126:42861: Configuration error: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), [1350ms] Sub RPC ConnectToMaster: received response from server master-127.23.218.124:32793: Configuration error: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), deferred=null) after lookup exception
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.23.218.125:44351,127.23.218.126:42861,127.23.218.124:32793). Exceptions received: [org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
at java.base/java.lang.Thread.run(Thread.java:840)
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.23.218.125:44351,127.23.218.126:42861,127.23.218.124:32793). Exceptions received: [org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
at java.base/java.lang.Thread.run(Thread.java:840)
08:04:29.650 [DEBUG - kudu-nio-2] (KuduRpc.java:260) Handling a callback on RPC KuduRpc(method=ListTabletServers, tablet=null, attempt=1, TimeoutTracker(timeout=30000, elapsed=1388), Traces: [0ms] refreshing cache from master, [72ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.218.125:44351, [287ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.218.126:42861, [306ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.218.124:32793, [1340ms] Sub RPC ConnectToMaster: received response from server master-127.23.218.125:44351: Configuration error: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), [1344ms] Sub RPC ConnectToMaster: received response from server master-127.23.218.126:42861: Configuration error: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), [1350ms] Sub RPC ConnectToMaster: received response from server master-127.23.218.124:32793: Configuration error: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), deferred=null) with no deferred attached!
08:04:29.653 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.23.218.126:42861(127.23.218.126:42861)] disconnecting while in state TERMINATED
08:04:29.654 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.23.218.125:44351(127.23.218.125:44351)] disconnecting while in state TERMINATED
08:04:29.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260430 08:04:29.654385 25561 negotiation.cc:336] Failed RPC negotiation. Trace:
08:04:29.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0430 08:04:28.649908 (+ 0us) reactor.cc:678] Submitting negotiation task for server connection from 127.0.0.1:46512 (local address 127.23.218.124:32793)
08:04:29.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0430 08:04:28.650658 (+ 750us) server_negotiation.cc:207] Beginning negotiation
08:04:29.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0430 08:04:28.650689 (+ 31us) server_negotiation.cc:400] Waiting for connection header
08:04:29.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0430 08:04:28.831343 (+180654us) server_negotiation.cc:408] Connection header received
08:04:29.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0430 08:04:28.873263 (+ 41920us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
08:04:29.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0430 08:04:28.873303 (+ 40us) server_negotiation.cc:462] Received NEGOTIATE request from client
08:04:29.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0430 08:04:28.873534 (+ 231us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
08:04:29.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0430 08:04:28.873721 (+ 187us) server_negotiation.cc:227] Negotiated authn=SASL
08:04:29.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0430 08:04:29.479224 (+605503us) server_negotiation.cc:366] Received TLS_HANDSHAKE NegotiatePB request
08:04:29.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0430 08:04:29.480549 (+ 1325us) server_negotiation.cc:378] Sending TLS_HANDSHAKE NegotiatePB response
08:04:29.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0430 08:04:29.608481 (+127932us) server_negotiation.cc:366] Received TLS_HANDSHAKE NegotiatePB request
08:04:29.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0430 08:04:29.609018 (+ 537us) server_negotiation.cc:663] Negotiated TLSv1.3 with cipher TLS_AES_128_GCM_SHA256 TLSv1.3 Kx=any Au=any Enc=AESGCM(128) Mac=AEAD
08:04:29.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0430 08:04:29.652996 (+ 43978us) negotiation.cc:326] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:46512: BlockingRecv error: failed to read from TLS socket (remote: 127.0.0.1:46512): Cannot send after transport endpoint shutdown (error 108)
08:04:29.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":92}
08:04:29.656 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.23.218.124:32793(127.23.218.124:32793)] disconnecting while in state TERMINATED
08:04:29.659 [DEBUG - Time-limited test] (AsyncKuduClient.java:2704) Releasing all remaining resources
08:04:29.659 [DEBUG - kudu-nio-2] (Connection.java:239) [peer master-127.23.218.124:32793(127.23.218.124:32793)] handling channelInactive
08:04:29.667 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
08:04:29.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:29.674474 24425 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskf7pa4T/build/tsan/bin/kudu with pid 24769
08:04:29.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:29.718279 24425 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskf7pa4T/build/tsan/bin/kudu with pid 24918
08:04:29.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:29.751834 24425 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskf7pa4T/build/tsan/bin/kudu with pid 25065
08:04:29.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:29.796556 24425 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskf7pa4T/build/tsan/bin/kudu with pid 25309
08:04:29.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:29.836300 24425 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskf7pa4T/build/tsan/bin/kudu with pid 25218
08:04:29.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:29.864470 24425 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskf7pa4T/build/tsan/bin/kudu with pid 25427
08:04:31.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-04-30T08:04:31Z chronyd exiting
08:04:32.070 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
08:04:32.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:32.073077 23696 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskf7pa4T/build/tsan/bin/kudu with pid 23977
08:04:32.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:32.099408 23696 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskf7pa4T/build/tsan/bin/kudu with pid 24141
08:04:32.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:32.132289 23696 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskf7pa4T/build/tsan/bin/kudu with pid 24283
08:04:32.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:32.160384 23696 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskf7pa4T/build/tsan/bin/kudu with pid 23708
08:04:32.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:32.189083 23696 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskf7pa4T/build/tsan/bin/kudu with pid 23778
08:04:32.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260430 08:04:32.222214 23696 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskf7pa4T/build/tsan/bin/kudu with pid 23853
08:04:33.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-04-30T08:04:33Z chronyd exiting
08:04:34.396 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.23.218.125:44351,127.23.218.126:42861,127.23.218.124:32793). Exceptions received: [org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)]
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:569)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at java.base/java.lang.Thread.run(Thread.java:840)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
... 1 more
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.23.218.125:44351,127.23.218.126:42861,127.23.218.124:32793). Exceptions received: [org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)]
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:569)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at java.base/java.lang.Thread.run(Thread.java:840)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
... 1 more