02:03:49.193 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
02:03:49.205 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskqQ5_Rr/test-tmp
02:03:49.210 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
02:03:49.213 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskqQ5_Rr/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
02:03:49.411 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster5755697716724601332"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
02:03:49.590 [DEBUG - main] (MiniKuduCluster.java:195) Response:
02:03:49.592 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
02:03:49.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:49.597481 28059 mini_oidc.cc:150] Starting JWKS server
02:03:49.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:49.601881 28059 webserver.cc:533] Webserver started at http://127.0.0.1:35195/ using document root <none> and password file <none>
02:03:49.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:49.602499 28059 mini_oidc.cc:186] Starting OIDC Discovery server
02:03:49.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:49.604718 28059 webserver.cc:533] Webserver started at http://127.0.0.1:39889/ using document root <none> and password file <none>
02:03:49.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-17T02:03:49Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
02:03:49.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-17T02:03:49Z Disabled control of system clock
02:03:49.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:49.670881 28059 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskqQ5_Rr/build/tsan/bin/kudu
02:03:49.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskqQ5_Rr/build/tsan/bin/kudu
02:03:49.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster5755697716724601332/master-0/wal
02:03:49.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster5755697716724601332/master-0/data
02:03:49.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:03:49.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:03:49.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:03:49.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:03:49.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:03:49.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:03:49.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster5755697716724601332/master-0/logs
02:03:49.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster5755697716724601332/master-0/data/info.pb
02:03:49.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:03:49.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:03:49.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:03:49.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:03:49.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:03:49.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:03:49.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:03:49.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:03:49.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:03:49.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:03:49.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:03:49.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:03:49.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.102.254:44423
02:03:49.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.102.254
02:03:49.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:03:49.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.102.212:34925
02:03:49.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:03:49.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:03:49.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:03:49.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:03:49.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.27.102.254:44423,127.27.102.253:36957,127.27.102.252:33421
02:03:49.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:03:49.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35195/jwks.json with env {}
02:03:49.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:03:49.982753 28071 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
02:03:49.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:03:49.983336 28071 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:03:49.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:03:49.983899 28071 flags.cc:432] Enabled unsafe flag: --never_fsync=true
02:03:50.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:03:50.016983 28071 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
02:03:50.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:03:50.017289 28071 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
02:03:50.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:03:50.017508 28071 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
02:03:50.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:03:50.017692 28071 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
02:03:50.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:03:50.017920 28071 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
02:03:50.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:03:50.018126 28071 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:35195/jwks.json
02:03:50.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:50.055632 28071 master_runner.cc:387] Master server non-default flags:
02:03:50.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:03:50.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.102.212:34925
02:03:50.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:03:50.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:03:50.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster5755697716724601332/master-0/data
02:03:50.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster5755697716724601332/master-0/wal
02:03:50.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:03:50.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.27.102.254:44423,127.27.102.253:36957,127.27.102.252:33421
02:03:50.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:03:50.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:03:50.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:03:50.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.102.254:44423
02:03:50.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:03:50.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:03:50.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:03:50.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35195/jwks.json
02:03:50.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:03:50.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:03:50.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster5755697716724601332/master-0/data/info.pb
02:03:50.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.102.254
02:03:50.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:03:50.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:03:50.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:03:50.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:03:50.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:03:50.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:03:50.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster5755697716724601332/master-0/logs
02:03:50.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:03:50.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:03:50.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:03:50.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:03:50.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:03:50.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 23b10557f461bc2be3d2c282748d8de96f9c46ec
02:03:50.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:03:50.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 17 Jun 2026 01:43:23 UTC on e7f111948823
02:03:50.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12431
02:03:50.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:03:50.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:50.057029 28071 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:03:50.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:50.058913 28071 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:03:50.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:03:50.071308 28077 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:03:51.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:03:50.071655 28078 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:03:51.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:03:51.300076 28080 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:03:51.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:03:51.303784 28079 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1227 milliseconds
02:03:51.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:03:51.304653 28071 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.233s user 0.432s sys 0.798s
02:03:51.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:03:51.305032 28071 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.233s user 0.432s sys 0.798s
02:03:51.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:51.305361 28071 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:03:51.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:51.307021 28071 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:03:51.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:51.310433 28071 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:03:51.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:51.314091 28071 hybrid_clock.cc:648] HybridClock initialized: now 1781661831313996 us; error 100 us; skew 500 ppm
02:03:51.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:51.315325 28071 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:03:51.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:51.323381 28071 webserver.cc:533] Webserver started at http://127.27.102.254:40841/ using document root <none> and password file <none>
02:03:51.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:51.324419 28071 fs_manager.cc:362] Metadata directory not provided
02:03:51.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:51.324690 28071 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:03:51.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:51.325174 28071 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
02:03:51.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:51.330111 28071 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster5755697716724601332/master-0/data/instance:
02:03:51.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "842b6c2a7322454eb92510275c056e25"
02:03:51.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-17 02:03:51 on dist-test-slave-32ct"
02:03:51.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:51.331460 28071 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster5755697716724601332/master-0/wal/instance:
02:03:51.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "842b6c2a7322454eb92510275c056e25"
02:03:51.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-17 02:03:51 on dist-test-slave-32ct"
02:03:51.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:51.340070 28071 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.006s sys 0.003s
02:03:51.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:51.346694 28087 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:03:51.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:51.348202 28071 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.004s sys 0.000s
02:03:51.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:51.348598 28071 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster5755697716724601332/master-0/data,/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster5755697716724601332/master-0/wal
02:03:51.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "842b6c2a7322454eb92510275c056e25"
02:03:51.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-17 02:03:51 on dist-test-slave-32ct"
02:03:51.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:51.349076 28071 fs_report.cc:389] FS layout report
02:03:51.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:03:51.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster5755697716724601332/master-0/wal
02:03:51.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster5755697716724601332/master-0/wal
02:03:51.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster5755697716724601332/master-0/data/data
02:03:51.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:03:51.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:03:51.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:03:51.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:03:51.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:03:51.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:03:51.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:03:51.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:03:51.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:03:51.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:03:51.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:03:51.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:51.421682 28071 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:03:51.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:51.423456 28071 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:03:51.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:51.424032 28071 kserver.cc:163] Server-wide thread pool size limit: 3276
02:03:51.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:51.498796 28071 rpc_server.cc:307] RPC server started. Bound to: 127.27.102.254:44423
02:03:51.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:51.498878 28138 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.27.102.254:44423 every 8 connection(s)
02:03:51.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:51.502003 28071 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster5755697716724601332/master-0/data/info.pb
02:03:51.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:51.506578 28059 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskqQ5_Rr/build/tsan/bin/kudu as pid 28071
02:03:51.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:51.507105 28059 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster5755697716724601332/master-0/wal/instance
02:03:51.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:51.509162 28139 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
02:03:51.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:51.512321 28059 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskqQ5_Rr/build/tsan/bin/kudu
02:03:51.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskqQ5_Rr/build/tsan/bin/kudu
02:03:51.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster5755697716724601332/master-1/wal
02:03:51.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster5755697716724601332/master-1/data
02:03:51.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:03:51.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:03:51.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:03:51.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:03:51.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:03:51.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:03:51.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster5755697716724601332/master-1/logs
02:03:51.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster5755697716724601332/master-1/data/info.pb
02:03:51.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:03:51.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:03:51.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:03:51.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:03:51.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:03:51.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:03:51.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:03:51.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:03:51.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:03:51.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:03:51.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:03:51.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:03:51.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.102.253:36957
02:03:51.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.102.253
02:03:51.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:03:51.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.102.212:34925
02:03:51.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:03:51.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:03:51.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:03:51.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:03:51.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.27.102.254:44423,127.27.102.253:36957,127.27.102.252:33421
02:03:51.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:03:51.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35195/jwks.json with env {}
02:03:51.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:51.527675 28139 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.102.254" port: 44423 } has no permanent_uuid. Determining permanent_uuid...
02:03:51.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:51.560647 28139 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.102.253" port: 36957 } has no permanent_uuid. Determining permanent_uuid...
02:03:51.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:03:51.563632 28088 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.27.102.253:36957: connect: Connection refused (error 111)
02:03:51.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:03:51.567945 28139 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.102.253:36957: Network error: Client connection negotiation failed: client connection to 127.27.102.253:36957: connect: Connection refused (error 111)
02:03:51.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:51.617496 28139 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.102.253" port: 36957 } attempt: 1
02:03:51.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:03:51.623095 28139 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.102.253:36957: Network error: Client connection negotiation failed: client connection to 127.27.102.253:36957: connect: Connection refused (error 111)
02:03:51.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:51.691524 28139 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.102.253" port: 36957 } attempt: 2
02:03:51.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:03:51.696274 28139 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.102.253:36957: Network error: Client connection negotiation failed: client connection to 127.27.102.253:36957: connect: Connection refused (error 111)
02:03:51.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:51.787698 28139 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.102.253" port: 36957 } attempt: 3
02:03:51.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:03:51.792460 28139 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.102.253:36957: Network error: Client connection negotiation failed: client connection to 127.27.102.253:36957: connect: Connection refused (error 111)
02:03:51.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:03:51.842060 28141 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
02:03:51.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:03:51.842705 28141 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:03:51.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:03:51.843217 28141 flags.cc:432] Enabled unsafe flag: --never_fsync=true
02:03:51.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:03:51.878271 28141 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
02:03:51.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:03:51.878567 28141 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
02:03:51.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:03:51.878772 28141 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
02:03:51.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:03:51.878978 28141 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
02:03:51.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:03:51.879163 28141 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
02:03:51.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:03:51.879302 28141 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:35195/jwks.json
02:03:51.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:51.918385 28141 master_runner.cc:387] Master server non-default flags:
02:03:51.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:03:51.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.102.212:34925
02:03:51.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:03:51.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:03:51.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster5755697716724601332/master-1/data
02:03:51.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster5755697716724601332/master-1/wal
02:03:51.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:03:51.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.27.102.254:44423,127.27.102.253:36957,127.27.102.252:33421
02:03:51.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:03:51.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:03:51.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:03:51.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.102.253:36957
02:03:51.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:03:51.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:03:51.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:03:51.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35195/jwks.json
02:03:51.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:03:51.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:03:51.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster5755697716724601332/master-1/data/info.pb
02:03:51.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.102.253
02:03:51.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:03:51.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:03:51.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:03:51.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:03:51.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:03:51.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:03:51.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster5755697716724601332/master-1/logs
02:03:51.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:03:51.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:03:51.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:03:51.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:03:51.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:03:51.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 23b10557f461bc2be3d2c282748d8de96f9c46ec
02:03:51.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:03:51.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 17 Jun 2026 01:43:23 UTC on e7f111948823
02:03:51.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12431
02:03:51.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:03:51.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:51.919834 28141 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:03:51.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:51.921821 28141 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:03:51.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:03:51.934633 28150 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:03:51.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:51.935880 28139 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.102.253" port: 36957 } attempt: 4
02:03:51.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:03:51.942402 28139 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.102.253:36957: Network error: Client connection negotiation failed: client connection to 127.27.102.253:36957: connect: Connection refused (error 111)
02:03:52.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:52.241946 28139 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.102.253" port: 36957 } attempt: 5
02:03:52.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:03:52.249117 28139 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.102.253:36957: Network error: Client connection negotiation failed: client connection to 127.27.102.253:36957: connect: Connection refused (error 111)
02:03:52.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:52.796700 28139 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.102.253" port: 36957 } attempt: 6
02:03:52.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:03:52.809289 28139 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.102.253:36957: Network error: Client connection negotiation failed: client connection to 127.27.102.253:36957: connect: Connection refused (error 111)
02:03:53.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:03:51.936400 28151 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:03:53.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:03:53.170383 28153 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:03:53.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:03:53.173951 28152 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1237 milliseconds
02:03:53.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:53.174033 28141 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:03:53.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:53.175666 28141 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:03:53.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:53.178959 28141 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:03:53.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:53.180372 28141 hybrid_clock.cc:648] HybridClock initialized: now 1781661833180332 us; error 64 us; skew 500 ppm
02:03:53.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:53.181185 28141 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:03:53.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:53.188004 28141 webserver.cc:533] Webserver started at http://127.27.102.253:33955/ using document root <none> and password file <none>
02:03:53.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:53.188903 28141 fs_manager.cc:362] Metadata directory not provided
02:03:53.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:53.189090 28141 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:03:53.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:53.189607 28141 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
02:03:53.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:53.194434 28141 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster5755697716724601332/master-1/data/instance:
02:03:53.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "89d379944adf49edb04c90a64ab032d0"
02:03:53.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-17 02:03:53 on dist-test-slave-32ct"
02:03:53.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:53.195731 28141 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster5755697716724601332/master-1/wal/instance:
02:03:53.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "89d379944adf49edb04c90a64ab032d0"
02:03:53.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-17 02:03:53 on dist-test-slave-32ct"
02:03:53.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:53.203261 28141 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.001s sys 0.007s
02:03:53.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:53.209468 28162 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:03:53.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:53.210641 28141 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
02:03:53.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:53.210933 28141 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster5755697716724601332/master-1/data,/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster5755697716724601332/master-1/wal
02:03:53.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "89d379944adf49edb04c90a64ab032d0"
02:03:53.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-17 02:03:53 on dist-test-slave-32ct"
02:03:53.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:53.211220 28141 fs_report.cc:389] FS layout report
02:03:53.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:03:53.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster5755697716724601332/master-1/wal
02:03:53.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster5755697716724601332/master-1/wal
02:03:53.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster5755697716724601332/master-1/data/data
02:03:53.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:03:53.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:03:53.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:03:53.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:03:53.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:03:53.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:03:53.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:03:53.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:03:53.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:03:53.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:03:53.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:03:53.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:53.264811 28141 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:03:53.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:53.266366 28141 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:03:53.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:53.266916 28141 kserver.cc:163] Server-wide thread pool size limit: 3276
02:03:53.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:53.344238 28141 rpc_server.cc:307] RPC server started. Bound to: 127.27.102.253:36957
02:03:53.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:53.344321 28213 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.27.102.253:36957 every 8 connection(s)
02:03:53.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:53.347944 28141 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster5755697716724601332/master-1/data/info.pb
02:03:53.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:53.353626 28214 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
02:03:53.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:53.357481 28059 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskqQ5_Rr/build/tsan/bin/kudu as pid 28141
02:03:53.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:53.357818 28059 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster5755697716724601332/master-1/wal/instance
02:03:53.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:53.359747 28059 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskqQ5_Rr/build/tsan/bin/kudu
02:03:53.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskqQ5_Rr/build/tsan/bin/kudu
02:03:53.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster5755697716724601332/master-2/wal
02:03:53.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster5755697716724601332/master-2/data
02:03:53.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:03:53.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:03:53.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:03:53.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:03:53.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:03:53.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:03:53.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster5755697716724601332/master-2/logs
02:03:53.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster5755697716724601332/master-2/data/info.pb
02:03:53.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:03:53.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:03:53.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:03:53.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:03:53.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:03:53.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:03:53.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:03:53.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:03:53.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:03:53.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:03:53.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:03:53.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:03:53.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.102.252:33421
02:03:53.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.102.252
02:03:53.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:03:53.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.102.212:34925
02:03:53.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:03:53.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:03:53.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:03:53.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:03:53.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.27.102.254:44423,127.27.102.253:36957,127.27.102.252:33421
02:03:53.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:03:53.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35195/jwks.json with env {}
02:03:53.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:53.366953 28214 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.102.254" port: 44423 } has no permanent_uuid. Determining permanent_uuid...
02:03:53.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:53.389978 28214 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.102.253" port: 36957 } has no permanent_uuid. Determining permanent_uuid...
02:03:53.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:53.405161 28214 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.102.252" port: 33421 } has no permanent_uuid. Determining permanent_uuid...
02:03:53.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:03:53.408015 28164 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.27.102.252:33421: connect: Connection refused (error 111)
02:03:53.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:03:53.412823 28214 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.102.252:33421: Network error: Client connection negotiation failed: client connection to 127.27.102.252:33421: connect: Connection refused (error 111)
02:03:53.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:53.462373 28214 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.102.252" port: 33421 } attempt: 1
02:03:53.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:03:53.467021 28214 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.102.252:33421: Network error: Client connection negotiation failed: client connection to 127.27.102.252:33421: connect: Connection refused (error 111)
02:03:53.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:53.535422 28214 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.102.252" port: 33421 } attempt: 2
02:03:53.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:03:53.539654 28214 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.102.252:33421: Network error: Client connection negotiation failed: client connection to 127.27.102.252:33421: connect: Connection refused (error 111)
02:03:53.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:53.631124 28214 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.102.252" port: 33421 } attempt: 3
02:03:53.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:03:53.636296 28214 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.102.252:33421: Network error: Client connection negotiation failed: client connection to 127.27.102.252:33421: connect: Connection refused (error 111)
02:03:53.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:03:53.688557 28216 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
02:03:53.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:03:53.689098 28216 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:03:53.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:03:53.689762 28216 flags.cc:432] Enabled unsafe flag: --never_fsync=true
02:03:53.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:03:53.722863 28216 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
02:03:53.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:03:53.723140 28216 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
02:03:53.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:03:53.723325 28216 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
02:03:53.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:03:53.723510 28216 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
02:03:53.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:03:53.723687 28216 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
02:03:53.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:03:53.723846 28216 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:35195/jwks.json
02:03:53.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:53.760375 28216 master_runner.cc:387] Master server non-default flags:
02:03:53.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:03:53.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.102.212:34925
02:03:53.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:03:53.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:03:53.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster5755697716724601332/master-2/data
02:03:53.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster5755697716724601332/master-2/wal
02:03:53.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:03:53.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.27.102.254:44423,127.27.102.253:36957,127.27.102.252:33421
02:03:53.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:03:53.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:03:53.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:03:53.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.102.252:33421
02:03:53.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:03:53.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:03:53.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:03:53.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35195/jwks.json
02:03:53.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:03:53.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:03:53.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster5755697716724601332/master-2/data/info.pb
02:03:53.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.102.252
02:03:53.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:03:53.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:03:53.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:03:53.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:03:53.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:03:53.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:03:53.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster5755697716724601332/master-2/logs
02:03:53.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:03:53.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:03:53.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:03:53.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:03:53.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:03:53.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 23b10557f461bc2be3d2c282748d8de96f9c46ec
02:03:53.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:03:53.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 17 Jun 2026 01:43:23 UTC on e7f111948823
02:03:53.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12431
02:03:53.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:03:53.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:53.762060 28216 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:03:53.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:53.764358 28216 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:03:53.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:03:53.778589 28226 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:03:53.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:53.779757 28214 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.102.252" port: 33421 } attempt: 4
02:03:53.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:03:53.787673 28214 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.102.252:33421: Network error: Client connection negotiation failed: client connection to 127.27.102.252:33421: connect: Connection refused (error 111)
02:03:53.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:53.869917 28139 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.102.253" port: 36957 } attempt: 7
02:03:53.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:53.902453 28139 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.102.252" port: 33421 } has no permanent_uuid. Determining permanent_uuid...
02:03:53.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:03:53.936913 28139 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.102.252:33421: Network error: Client connection negotiation failed: client connection to 127.27.102.252:33421: connect: Connection refused (error 111)
02:03:53.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:53.995482 28139 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.102.252" port: 33421 } attempt: 1
02:03:54.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:03:54.003954 28139 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.102.252:33421: Network error: Client connection negotiation failed: client connection to 127.27.102.252:33421: connect: Connection refused (error 111)
02:03:54.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:54.085518 28139 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.102.252" port: 33421 } attempt: 2
02:03:54.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:54.087179 28214 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.102.252" port: 33421 } attempt: 5
02:03:54.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:03:54.095000 28214 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.102.252:33421: Network error: Client connection negotiation failed: client connection to 127.27.102.252:33421: connect: Connection refused (error 111)
02:03:54.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:03:54.101888 28139 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.102.252:33421: Network error: Client connection negotiation failed: client connection to 127.27.102.252:33421: connect: Connection refused (error 111)
02:03:54.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:54.187479 28139 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.102.252" port: 33421 } attempt: 3
02:03:54.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:03:54.195969 28139 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.102.252:33421: Network error: Client connection negotiation failed: client connection to 127.27.102.252:33421: connect: Connection refused (error 111)
02:03:54.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:54.336510 28139 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.102.252" port: 33421 } attempt: 4
02:03:54.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:03:54.362167 28139 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.102.252:33421: Network error: Client connection negotiation failed: client connection to 127.27.102.252:33421: connect: Connection refused (error 111)
02:03:54.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:54.642551 28214 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.102.252" port: 33421 } attempt: 6
02:03:54.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:54.646152 28139 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.102.252" port: 33421 } attempt: 5
02:03:54.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:03:54.655238 28139 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.102.252:33421: Network error: Client connection negotiation failed: client connection to 127.27.102.252:33421: connect: Connection refused (error 111)
02:03:54.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:03:54.680219 28214 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.102.252:33421: Network error: Client connection negotiation failed: client connection to 127.27.102.252:33421: connect: Connection refused (error 111)
02:03:55.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:55.207767 28139 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.102.252" port: 33421 } attempt: 6
02:03:55.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:03:55.238956 28139 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.102.252:33421: Network error: Client connection negotiation failed: client connection to 127.27.102.252:33421: connect: Connection refused (error 111)
02:03:55.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:03:53.779212 28227 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:03:55.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:03:55.181689 28225 debug-util.cc:398] Leaking SignalData structure 0x7b0800028020 after lost signal to thread 28216
02:03:55.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:03:55.575423 28216 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.797s user 0.611s sys 1.176s
02:03:55.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:03:55.576890 28216 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.799s user 0.611s sys 1.176s
02:03:55.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:03:55.577338 28228 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1798 milliseconds
02:03:55.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:03:55.577795 28229 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:03:55.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:55.577840 28216 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:03:55.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:55.581664 28216 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:03:55.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:55.584082 28216 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:03:55.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:55.585459 28216 hybrid_clock.cc:648] HybridClock initialized: now 1781661835585436 us; error 61 us; skew 500 ppm
02:03:55.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:55.586256 28216 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:03:55.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:55.592193 28216 webserver.cc:533] Webserver started at http://127.27.102.252:41293/ using document root <none> and password file <none>
02:03:55.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:55.593163 28216 fs_manager.cc:362] Metadata directory not provided
02:03:55.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:55.593380 28216 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:03:55.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:55.593804 28216 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
02:03:55.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:55.598014 28216 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster5755697716724601332/master-2/data/instance:
02:03:55.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2a9e163c188b436482ef90a861478b6e"
02:03:55.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-17 02:03:55 on dist-test-slave-32ct"
02:03:55.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:55.599318 28216 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster5755697716724601332/master-2/wal/instance:
02:03:55.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2a9e163c188b436482ef90a861478b6e"
02:03:55.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-17 02:03:55 on dist-test-slave-32ct"
02:03:55.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:55.606927 28216 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.004s sys 0.004s
02:03:55.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:55.612597 28243 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:03:55.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:55.613782 28216 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.001s
02:03:55.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:55.614135 28216 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster5755697716724601332/master-2/data,/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster5755697716724601332/master-2/wal
02:03:55.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2a9e163c188b436482ef90a861478b6e"
02:03:55.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-17 02:03:55 on dist-test-slave-32ct"
02:03:55.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:55.614527 28216 fs_report.cc:389] FS layout report
02:03:55.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:03:55.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster5755697716724601332/master-2/wal
02:03:55.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster5755697716724601332/master-2/wal
02:03:55.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster5755697716724601332/master-2/data/data
02:03:55.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:03:55.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:03:55.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:03:55.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:03:55.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:03:55.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:03:55.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:03:55.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:03:55.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:03:55.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:03:55.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:03:55.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:55.665684 28216 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:03:55.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:55.667244 28216 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:03:55.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:55.667779 28216 kserver.cc:163] Server-wide thread pool size limit: 3276
02:03:55.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:55.737160 28216 rpc_server.cc:307] RPC server started. Bound to: 127.27.102.252:33421
02:03:55.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:55.737247 28294 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.27.102.252:33421 every 8 connection(s)
02:03:55.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:55.740038 28216 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster5755697716724601332/master-2/data/info.pb
02:03:55.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:55.740758 28214 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.102.252" port: 33421 } attempt: 7
02:03:55.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:55.746308 28059 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskqQ5_Rr/build/tsan/bin/kudu as pid 28216
02:03:55.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:55.746778 28059 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster5755697716724601332/master-2/wal/instance
02:03:55.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:55.748041 28295 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
02:03:55.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:55.774245 28295 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.102.254" port: 44423 } has no permanent_uuid. Determining permanent_uuid...
02:03:55.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:55.794114 28295 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.102.253" port: 36957 } has no permanent_uuid. Determining permanent_uuid...
02:03:55.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:55.803555 28214 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 89d379944adf49edb04c90a64ab032d0: Bootstrap starting.
02:03:55.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:55.808327 28295 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.102.252" port: 33421 } has no permanent_uuid. Determining permanent_uuid...
02:03:55.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:55.810568 28214 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 89d379944adf49edb04c90a64ab032d0: Neither blocks nor log segments found. Creating new log.
02:03:55.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:55.815510 28214 log.cc:826] T 00000000000000000000000000000000 P 89d379944adf49edb04c90a64ab032d0: Log is configured to *not* fsync() on all Append() calls
02:03:55.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:55.820816 28214 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 89d379944adf49edb04c90a64ab032d0: No bootstrap required, opened a new log
02:03:55.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:55.826613 28295 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 2a9e163c188b436482ef90a861478b6e: Bootstrap starting.
02:03:55.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:55.833683 28295 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 2a9e163c188b436482ef90a861478b6e: Neither blocks nor log segments found. Creating new log.
02:03:55.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:55.835467 28295 log.cc:826] T 00000000000000000000000000000000 P 2a9e163c188b436482ef90a861478b6e: Log is configured to *not* fsync() on all Append() calls
02:03:55.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:55.842681 28295 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 2a9e163c188b436482ef90a861478b6e: No bootstrap required, opened a new log
02:03:55.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:55.849920 28214 raft_consensus.cc:359] T 00000000000000000000000000000000 P 89d379944adf49edb04c90a64ab032d0 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "842b6c2a7322454eb92510275c056e25" member_type: VOTER last_known_addr { host: "127.27.102.254" port: 44423 } } peers { permanent_uuid: "89d379944adf49edb04c90a64ab032d0" member_type: VOTER last_known_addr { host: "127.27.102.253" port: 36957 } } peers { permanent_uuid: "2a9e163c188b436482ef90a861478b6e" member_type: VOTER last_known_addr { host: "127.27.102.252" port: 33421 } }
02:03:55.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:55.850576 28214 raft_consensus.cc:385] T 00000000000000000000000000000000 P 89d379944adf49edb04c90a64ab032d0 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:03:55.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:55.850926 28214 raft_consensus.cc:740] T 00000000000000000000000000000000 P 89d379944adf49edb04c90a64ab032d0 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 89d379944adf49edb04c90a64ab032d0, State: Initialized, Role: FOLLOWER
02:03:55.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:55.851691 28214 consensus_queue.cc:260] T 00000000000000000000000000000000 P 89d379944adf49edb04c90a64ab032d0 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "842b6c2a7322454eb92510275c056e25" member_type: VOTER last_known_addr { host: "127.27.102.254" port: 44423 } } peers { permanent_uuid: "89d379944adf49edb04c90a64ab032d0" member_type: VOTER last_known_addr { host: "127.27.102.253" port: 36957 } } peers { permanent_uuid: "2a9e163c188b436482ef90a861478b6e" member_type: VOTER last_known_addr { host: "127.27.102.252" port: 33421 } }
02:03:55.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:55.854194 28308 sys_catalog.cc:455] T 00000000000000000000000000000000 P 89d379944adf49edb04c90a64ab032d0 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "842b6c2a7322454eb92510275c056e25" member_type: VOTER last_known_addr { host: "127.27.102.254" port: 44423 } } peers { permanent_uuid: "89d379944adf49edb04c90a64ab032d0" member_type: VOTER last_known_addr { host: "127.27.102.253" port: 36957 } } peers { permanent_uuid: "2a9e163c188b436482ef90a861478b6e" member_type: VOTER last_known_addr { host: "127.27.102.252" port: 33421 } } }
02:03:55.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:55.854979 28308 sys_catalog.cc:458] T 00000000000000000000000000000000 P 89d379944adf49edb04c90a64ab032d0 [sys.catalog]: This master's current role is: FOLLOWER
02:03:55.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:55.855540 28214 sys_catalog.cc:565] T 00000000000000000000000000000000 P 89d379944adf49edb04c90a64ab032d0 [sys.catalog]: configured and running, proceeding with master startup.
02:03:55.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:55.865769 28295 raft_consensus.cc:359] T 00000000000000000000000000000000 P 2a9e163c188b436482ef90a861478b6e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "842b6c2a7322454eb92510275c056e25" member_type: VOTER last_known_addr { host: "127.27.102.254" port: 44423 } } peers { permanent_uuid: "89d379944adf49edb04c90a64ab032d0" member_type: VOTER last_known_addr { host: "127.27.102.253" port: 36957 } } peers { permanent_uuid: "2a9e163c188b436482ef90a861478b6e" member_type: VOTER last_known_addr { host: "127.27.102.252" port: 33421 } }
02:03:55.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:55.867282 28295 raft_consensus.cc:385] T 00000000000000000000000000000000 P 2a9e163c188b436482ef90a861478b6e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:03:55.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:55.867544 28295 raft_consensus.cc:740] T 00000000000000000000000000000000 P 2a9e163c188b436482ef90a861478b6e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 2a9e163c188b436482ef90a861478b6e, State: Initialized, Role: FOLLOWER
02:03:55.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:55.869798 28295 consensus_queue.cc:260] T 00000000000000000000000000000000 P 2a9e163c188b436482ef90a861478b6e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "842b6c2a7322454eb92510275c056e25" member_type: VOTER last_known_addr { host: "127.27.102.254" port: 44423 } } peers { permanent_uuid: "89d379944adf49edb04c90a64ab032d0" member_type: VOTER last_known_addr { host: "127.27.102.253" port: 36957 } } peers { permanent_uuid: "2a9e163c188b436482ef90a861478b6e" member_type: VOTER last_known_addr { host: "127.27.102.252" port: 33421 } }
02:03:55.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:55.873222 28315 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2a9e163c188b436482ef90a861478b6e [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "842b6c2a7322454eb92510275c056e25" member_type: VOTER last_known_addr { host: "127.27.102.254" port: 44423 } } peers { permanent_uuid: "89d379944adf49edb04c90a64ab032d0" member_type: VOTER last_known_addr { host: "127.27.102.253" port: 36957 } } peers { permanent_uuid: "2a9e163c188b436482ef90a861478b6e" member_type: VOTER last_known_addr { host: "127.27.102.252" port: 33421 } } }
02:03:55.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:55.874212 28315 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2a9e163c188b436482ef90a861478b6e [sys.catalog]: This master's current role is: FOLLOWER
02:03:55.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:55.875108 28295 sys_catalog.cc:565] T 00000000000000000000000000000000 P 2a9e163c188b436482ef90a861478b6e [sys.catalog]: configured and running, proceeding with master startup.
02:03:55.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:03:55.889580 28322 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 89d379944adf49edb04c90a64ab032d0: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:03:55.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:03:55.890331 28322 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:03:55.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:55.886675 28141 master_runner.cc:187] Error getting master registration for 127.27.102.254:44423: OK, instance_id { permanent_uuid: "842b6c2a7322454eb92510275c056e25" instance_seqno: 1781661831478324 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
02:03:55.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:55.920387 28141 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
02:03:55.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:03:55.922773 28249 tablet.cc:2406] T 00000000000000000000000000000000 P 2a9e163c188b436482ef90a861478b6e: Can't schedule compaction. Clean time has not been advanced past its initial value.
02:03:55.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:03:55.925716 28332 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 2a9e163c188b436482ef90a861478b6e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:03:55.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:03:55.926309 28332 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:03:55.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:55.913827 28216 master_runner.cc:187] Error getting master registration for 127.27.102.254:44423: OK, instance_id { permanent_uuid: "842b6c2a7322454eb92510275c056e25" instance_seqno: 1781661831478324 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
02:03:55.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:55.945376 28216 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
02:03:55.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:55.946414 28315 raft_consensus.cc:493] T 00000000000000000000000000000000 P 2a9e163c188b436482ef90a861478b6e [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:03:55.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:55.947021 28315 raft_consensus.cc:515] T 00000000000000000000000000000000 P 2a9e163c188b436482ef90a861478b6e [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "842b6c2a7322454eb92510275c056e25" member_type: VOTER last_known_addr { host: "127.27.102.254" port: 44423 } } peers { permanent_uuid: "89d379944adf49edb04c90a64ab032d0" member_type: VOTER last_known_addr { host: "127.27.102.253" port: 36957 } } peers { permanent_uuid: "2a9e163c188b436482ef90a861478b6e" member_type: VOTER last_known_addr { host: "127.27.102.252" port: 33421 } }
02:03:55.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:55.949946 28308 raft_consensus.cc:493] T 00000000000000000000000000000000 P 89d379944adf49edb04c90a64ab032d0 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:03:55.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:55.950610 28308 raft_consensus.cc:515] T 00000000000000000000000000000000 P 89d379944adf49edb04c90a64ab032d0 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "842b6c2a7322454eb92510275c056e25" member_type: VOTER last_known_addr { host: "127.27.102.254" port: 44423 } } peers { permanent_uuid: "89d379944adf49edb04c90a64ab032d0" member_type: VOTER last_known_addr { host: "127.27.102.253" port: 36957 } } peers { permanent_uuid: "2a9e163c188b436482ef90a861478b6e" member_type: VOTER last_known_addr { host: "127.27.102.252" port: 33421 } }
02:03:55.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:55.951447 28315 leader_election.cc:290] T 00000000000000000000000000000000 P 2a9e163c188b436482ef90a861478b6e [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 842b6c2a7322454eb92510275c056e25 (127.27.102.254:44423), 89d379944adf49edb04c90a64ab032d0 (127.27.102.253:36957)
02:03:55.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:55.957741 28270 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "89d379944adf49edb04c90a64ab032d0" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "2a9e163c188b436482ef90a861478b6e" is_pre_election: true
02:03:55.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:55.958513 28270 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 2a9e163c188b436482ef90a861478b6e [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 89d379944adf49edb04c90a64ab032d0 in term 0.
02:03:55.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:55.959527 28189 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "2a9e163c188b436482ef90a861478b6e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "89d379944adf49edb04c90a64ab032d0" is_pre_election: true
02:03:55.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:55.961251 28189 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 89d379944adf49edb04c90a64ab032d0 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 2a9e163c188b436482ef90a861478b6e in term 0.
02:03:55.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:55.965704 28164 leader_election.cc:304] T 00000000000000000000000000000000 P 89d379944adf49edb04c90a64ab032d0 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 2a9e163c188b436482ef90a861478b6e, 89d379944adf49edb04c90a64ab032d0; no voters:
02:03:55.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:55.958922 28308 leader_election.cc:290] T 00000000000000000000000000000000 P 89d379944adf49edb04c90a64ab032d0 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 842b6c2a7322454eb92510275c056e25 (127.27.102.254:44423), 2a9e163c188b436482ef90a861478b6e (127.27.102.252:33421)
02:03:55.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:55.968887 28244 leader_election.cc:304] T 00000000000000000000000000000000 P 2a9e163c188b436482ef90a861478b6e [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 2a9e163c188b436482ef90a861478b6e, 89d379944adf49edb04c90a64ab032d0; no voters:
02:03:55.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:55.969130 28308 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 89d379944adf49edb04c90a64ab032d0 [term 0 FOLLOWER]: Leader pre-election won for term 1
02:03:55.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:55.969828 28315 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 2a9e163c188b436482ef90a861478b6e [term 0 FOLLOWER]: Leader pre-election won for term 1
02:03:55.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:55.970157 28315 raft_consensus.cc:493] T 00000000000000000000000000000000 P 2a9e163c188b436482ef90a861478b6e [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
02:03:55.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:55.969973 28308 raft_consensus.cc:493] T 00000000000000000000000000000000 P 89d379944adf49edb04c90a64ab032d0 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
02:03:55.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:55.970459 28315 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 2a9e163c188b436482ef90a861478b6e [term 0 FOLLOWER]: Advancing to term 1
02:03:55.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:55.970723 28308 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 89d379944adf49edb04c90a64ab032d0 [term 0 FOLLOWER]: Advancing to term 1
02:03:55.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:55.989027 28315 raft_consensus.cc:515] T 00000000000000000000000000000000 P 2a9e163c188b436482ef90a861478b6e [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "842b6c2a7322454eb92510275c056e25" member_type: VOTER last_known_addr { host: "127.27.102.254" port: 44423 } } peers { permanent_uuid: "89d379944adf49edb04c90a64ab032d0" member_type: VOTER last_known_addr { host: "127.27.102.253" port: 36957 } } peers { permanent_uuid: "2a9e163c188b436482ef90a861478b6e" member_type: VOTER last_known_addr { host: "127.27.102.252" port: 33421 } }
02:03:56.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:55.997479 28308 raft_consensus.cc:515] T 00000000000000000000000000000000 P 89d379944adf49edb04c90a64ab032d0 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "842b6c2a7322454eb92510275c056e25" member_type: VOTER last_known_addr { host: "127.27.102.254" port: 44423 } } peers { permanent_uuid: "89d379944adf49edb04c90a64ab032d0" member_type: VOTER last_known_addr { host: "127.27.102.253" port: 36957 } } peers { permanent_uuid: "2a9e163c188b436482ef90a861478b6e" member_type: VOTER last_known_addr { host: "127.27.102.252" port: 33421 } }
02:03:56.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:56.004880 28189 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "2a9e163c188b436482ef90a861478b6e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "89d379944adf49edb04c90a64ab032d0"
02:03:56.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:56.007980 28308 leader_election.cc:290] T 00000000000000000000000000000000 P 89d379944adf49edb04c90a64ab032d0 [CANDIDATE]: Term 1 election: Requested vote from peers 842b6c2a7322454eb92510275c056e25 (127.27.102.254:44423), 2a9e163c188b436482ef90a861478b6e (127.27.102.252:33421)
02:03:56.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:56.009825 28189 raft_consensus.cc:2393] T 00000000000000000000000000000000 P 89d379944adf49edb04c90a64ab032d0 [term 1 FOLLOWER]: Leader election vote request: Denying vote to candidate 2a9e163c188b436482ef90a861478b6e in current term 1: Already voted for candidate 89d379944adf49edb04c90a64ab032d0 in this term.
02:03:56.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:56.012287 28315 leader_election.cc:290] T 00000000000000000000000000000000 P 2a9e163c188b436482ef90a861478b6e [CANDIDATE]: Term 1 election: Requested vote from peers 842b6c2a7322454eb92510275c056e25 (127.27.102.254:44423), 89d379944adf49edb04c90a64ab032d0 (127.27.102.253:36957)
02:03:56.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:03:56.014915 28164 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.RequestConsensusVote -> {remote=127.27.102.252:33421, user_credentials={real_user=slave}} blocked reactor thread for 49279.3us
02:03:56.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:56.019012 28270 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "89d379944adf49edb04c90a64ab032d0" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "2a9e163c188b436482ef90a861478b6e"
02:03:56.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:56.019829 28270 raft_consensus.cc:2393] T 00000000000000000000000000000000 P 2a9e163c188b436482ef90a861478b6e [term 1 FOLLOWER]: Leader election vote request: Denying vote to candidate 89d379944adf49edb04c90a64ab032d0 in current term 1: Already voted for candidate 2a9e163c188b436482ef90a861478b6e in this term.
02:03:56.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:03:56.033396 28168 tablet.cc:2406] T 00000000000000000000000000000000 P 89d379944adf49edb04c90a64ab032d0: Can't schedule compaction. Clean time has not been advanced past its initial value.
02:03:56.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:56.272471 28139 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.102.252" port: 33421 } attempt: 7
02:03:56.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:56.311231 28139 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 842b6c2a7322454eb92510275c056e25: Bootstrap starting.
02:03:56.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:56.318496 28139 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 842b6c2a7322454eb92510275c056e25: Neither blocks nor log segments found. Creating new log.
02:03:56.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:56.321131 28139 log.cc:826] T 00000000000000000000000000000000 P 842b6c2a7322454eb92510275c056e25: Log is configured to *not* fsync() on all Append() calls
02:03:56.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:56.327845 28139 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 842b6c2a7322454eb92510275c056e25: No bootstrap required, opened a new log
02:03:56.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:56.359576 28139 raft_consensus.cc:359] T 00000000000000000000000000000000 P 842b6c2a7322454eb92510275c056e25 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "842b6c2a7322454eb92510275c056e25" member_type: VOTER last_known_addr { host: "127.27.102.254" port: 44423 } } peers { permanent_uuid: "89d379944adf49edb04c90a64ab032d0" member_type: VOTER last_known_addr { host: "127.27.102.253" port: 36957 } } peers { permanent_uuid: "2a9e163c188b436482ef90a861478b6e" member_type: VOTER last_known_addr { host: "127.27.102.252" port: 33421 } }
02:03:56.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:56.360872 28139 raft_consensus.cc:385] T 00000000000000000000000000000000 P 842b6c2a7322454eb92510275c056e25 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:03:56.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:56.361470 28139 raft_consensus.cc:740] T 00000000000000000000000000000000 P 842b6c2a7322454eb92510275c056e25 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 842b6c2a7322454eb92510275c056e25, State: Initialized, Role: FOLLOWER
02:03:56.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:56.365989 28139 consensus_queue.cc:260] T 00000000000000000000000000000000 P 842b6c2a7322454eb92510275c056e25 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "842b6c2a7322454eb92510275c056e25" member_type: VOTER last_known_addr { host: "127.27.102.254" port: 44423 } } peers { permanent_uuid: "89d379944adf49edb04c90a64ab032d0" member_type: VOTER last_known_addr { host: "127.27.102.253" port: 36957 } } peers { permanent_uuid: "2a9e163c188b436482ef90a861478b6e" member_type: VOTER last_known_addr { host: "127.27.102.252" port: 33421 } }
02:03:56.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:56.386394 28114 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "2a9e163c188b436482ef90a861478b6e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "842b6c2a7322454eb92510275c056e25" is_pre_election: true
02:03:56.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:56.387188 28114 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 842b6c2a7322454eb92510275c056e25 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 2a9e163c188b436482ef90a861478b6e in term 0.
02:03:56.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:56.388180 28111 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "89d379944adf49edb04c90a64ab032d0" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "842b6c2a7322454eb92510275c056e25"
02:03:56.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:56.388716 28111 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 842b6c2a7322454eb92510275c056e25 [term 0 FOLLOWER]: Advancing to term 1
02:03:56.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:56.393563 28112 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "2a9e163c188b436482ef90a861478b6e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "842b6c2a7322454eb92510275c056e25"
02:03:56.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:56.394812 28113 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "89d379944adf49edb04c90a64ab032d0" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "842b6c2a7322454eb92510275c056e25" is_pre_election: true
02:03:56.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:56.398854 28247 leader_election.cc:304] T 00000000000000000000000000000000 P 2a9e163c188b436482ef90a861478b6e [CANDIDATE]: Term 1 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 2a9e163c188b436482ef90a861478b6e; no voters: 842b6c2a7322454eb92510275c056e25, 89d379944adf49edb04c90a64ab032d0
02:03:56.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:56.400272 28315 raft_consensus.cc:2749] T 00000000000000000000000000000000 P 2a9e163c188b436482ef90a861478b6e [term 1 FOLLOWER]: Leader election lost for term 1. Reason: could not achieve majority
02:03:56.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:56.403194 28111 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 842b6c2a7322454eb92510275c056e25 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 89d379944adf49edb04c90a64ab032d0 in term 1.
02:03:56.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:56.385255 28337 sys_catalog.cc:455] T 00000000000000000000000000000000 P 842b6c2a7322454eb92510275c056e25 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "842b6c2a7322454eb92510275c056e25" member_type: VOTER last_known_addr { host: "127.27.102.254" port: 44423 } } peers { permanent_uuid: "89d379944adf49edb04c90a64ab032d0" member_type: VOTER last_known_addr { host: "127.27.102.253" port: 36957 } } peers { permanent_uuid: "2a9e163c188b436482ef90a861478b6e" member_type: VOTER last_known_addr { host: "127.27.102.252" port: 33421 } } }
02:03:56.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:56.404531 28337 sys_catalog.cc:458] T 00000000000000000000000000000000 P 842b6c2a7322454eb92510275c056e25 [sys.catalog]: This master's current role is: FOLLOWER
02:03:56.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:56.404421 28166 leader_election.cc:304] T 00000000000000000000000000000000 P 89d379944adf49edb04c90a64ab032d0 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 842b6c2a7322454eb92510275c056e25, 89d379944adf49edb04c90a64ab032d0; no voters: 2a9e163c188b436482ef90a861478b6e
02:03:56.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:56.404937 28139 sys_catalog.cc:565] T 00000000000000000000000000000000 P 842b6c2a7322454eb92510275c056e25 [sys.catalog]: configured and running, proceeding with master startup.
02:03:56.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:56.405625 28334 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 89d379944adf49edb04c90a64ab032d0 [term 1 FOLLOWER]: Leader election won for term 1
02:03:56.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:56.406112 28334 raft_consensus.cc:697] T 00000000000000000000000000000000 P 89d379944adf49edb04c90a64ab032d0 [term 1 LEADER]: Becoming Leader. State: Replica: 89d379944adf49edb04c90a64ab032d0, State: Running, Role: LEADER
02:03:56.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:56.407002 28334 consensus_queue.cc:237] T 00000000000000000000000000000000 P 89d379944adf49edb04c90a64ab032d0 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "842b6c2a7322454eb92510275c056e25" member_type: VOTER last_known_addr { host: "127.27.102.254" port: 44423 } } peers { permanent_uuid: "89d379944adf49edb04c90a64ab032d0" member_type: VOTER last_known_addr { host: "127.27.102.253" port: 36957 } } peers { permanent_uuid: "2a9e163c188b436482ef90a861478b6e" member_type: VOTER last_known_addr { host: "127.27.102.252" port: 33421 } }
02:03:56.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:03:56.435366 28093 tablet.cc:2406] T 00000000000000000000000000000000 P 842b6c2a7322454eb92510275c056e25: Can't schedule compaction. Clean time has not been advanced past its initial value.
02:03:56.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:56.437321 28308 sys_catalog.cc:455] T 00000000000000000000000000000000 P 89d379944adf49edb04c90a64ab032d0 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 89d379944adf49edb04c90a64ab032d0. Latest consensus state: current_term: 1 leader_uuid: "89d379944adf49edb04c90a64ab032d0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "842b6c2a7322454eb92510275c056e25" member_type: VOTER last_known_addr { host: "127.27.102.254" port: 44423 } } peers { permanent_uuid: "89d379944adf49edb04c90a64ab032d0" member_type: VOTER last_known_addr { host: "127.27.102.253" port: 36957 } } peers { permanent_uuid: "2a9e163c188b436482ef90a861478b6e" member_type: VOTER last_known_addr { host: "127.27.102.252" port: 33421 } } }
02:03:56.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:56.438416 28308 sys_catalog.cc:458] T 00000000000000000000000000000000 P 89d379944adf49edb04c90a64ab032d0 [sys.catalog]: This master's current role is: LEADER
02:03:56.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:56.445950 28342 catalog_manager.cc:1510] Loading table and tablet metadata into memory...
02:03:56.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:56.453508 28342 catalog_manager.cc:1519] Initializing Kudu cluster ID...
02:03:56.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:56.484318 28111 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 842b6c2a7322454eb92510275c056e25 [term 1 FOLLOWER]: Refusing update from remote peer 89d379944adf49edb04c90a64ab032d0: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
02:03:56.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:56.484961 28270 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 2a9e163c188b436482ef90a861478b6e [term 1 FOLLOWER]: Refusing update from remote peer 89d379944adf49edb04c90a64ab032d0: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
02:03:56.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:56.486446 28308 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 89d379944adf49edb04c90a64ab032d0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "842b6c2a7322454eb92510275c056e25" member_type: VOTER last_known_addr { host: "127.27.102.254" port: 44423 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
02:03:56.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:56.490509 28308 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 89d379944adf49edb04c90a64ab032d0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "2a9e163c188b436482ef90a861478b6e" member_type: VOTER last_known_addr { host: "127.27.102.252" port: 33421 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
02:03:56.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:56.541175 28344 mvcc.cc:204] Tried to move back new op lower bound from 7297686882220175360 to 7297686881932468224. Current Snapshot: MvccSnapshot[applied={T|T < 7297686882220175360}]
02:03:56.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:56.543159 28337 sys_catalog.cc:455] T 00000000000000000000000000000000 P 842b6c2a7322454eb92510275c056e25 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 89d379944adf49edb04c90a64ab032d0. Latest consensus state: current_term: 1 leader_uuid: "89d379944adf49edb04c90a64ab032d0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "842b6c2a7322454eb92510275c056e25" member_type: VOTER last_known_addr { host: "127.27.102.254" port: 44423 } } peers { permanent_uuid: "89d379944adf49edb04c90a64ab032d0" member_type: VOTER last_known_addr { host: "127.27.102.253" port: 36957 } } peers { permanent_uuid: "2a9e163c188b436482ef90a861478b6e" member_type: VOTER last_known_addr { host: "127.27.102.252" port: 33421 } } }
02:03:56.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:56.543846 28337 sys_catalog.cc:458] T 00000000000000000000000000000000 P 842b6c2a7322454eb92510275c056e25 [sys.catalog]: This master's current role is: FOLLOWER
02:03:56.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:56.542910 28315 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2a9e163c188b436482ef90a861478b6e [sys.catalog]: SysCatalogTable state changed. Reason: New leader 89d379944adf49edb04c90a64ab032d0. Latest consensus state: current_term: 1 leader_uuid: "89d379944adf49edb04c90a64ab032d0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "842b6c2a7322454eb92510275c056e25" member_type: VOTER last_known_addr { host: "127.27.102.254" port: 44423 } } peers { permanent_uuid: "89d379944adf49edb04c90a64ab032d0" member_type: VOTER last_known_addr { host: "127.27.102.253" port: 36957 } } peers { permanent_uuid: "2a9e163c188b436482ef90a861478b6e" member_type: VOTER last_known_addr { host: "127.27.102.252" port: 33421 } } }
02:03:56.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:56.546479 28315 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2a9e163c188b436482ef90a861478b6e [sys.catalog]: This master's current role is: FOLLOWER
02:03:56.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:56.552989 28352 mvcc.cc:204] Tried to move back new op lower bound from 7297686882220175360 to 7297686881932468224. Current Snapshot: MvccSnapshot[applied={T|T < 7297686882220175360}]
02:03:56.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:56.560295 28334 sys_catalog.cc:455] T 00000000000000000000000000000000 P 89d379944adf49edb04c90a64ab032d0 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "89d379944adf49edb04c90a64ab032d0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "842b6c2a7322454eb92510275c056e25" member_type: VOTER last_known_addr { host: "127.27.102.254" port: 44423 } } peers { permanent_uuid: "89d379944adf49edb04c90a64ab032d0" member_type: VOTER last_known_addr { host: "127.27.102.253" port: 36957 } } peers { permanent_uuid: "2a9e163c188b436482ef90a861478b6e" member_type: VOTER last_known_addr { host: "127.27.102.252" port: 33421 } } }
02:03:56.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:56.561235 28334 sys_catalog.cc:458] T 00000000000000000000000000000000 P 89d379944adf49edb04c90a64ab032d0 [sys.catalog]: This master's current role is: LEADER
02:03:56.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:56.563169 28334 sys_catalog.cc:455] T 00000000000000000000000000000000 P 89d379944adf49edb04c90a64ab032d0 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "89d379944adf49edb04c90a64ab032d0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "842b6c2a7322454eb92510275c056e25" member_type: VOTER last_known_addr { host: "127.27.102.254" port: 44423 } } peers { permanent_uuid: "89d379944adf49edb04c90a64ab032d0" member_type: VOTER last_known_addr { host: "127.27.102.253" port: 36957 } } peers { permanent_uuid: "2a9e163c188b436482ef90a861478b6e" member_type: VOTER last_known_addr { host: "127.27.102.252" port: 33421 } } }
02:03:56.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:56.563964 28334 sys_catalog.cc:458] T 00000000000000000000000000000000 P 89d379944adf49edb04c90a64ab032d0 [sys.catalog]: This master's current role is: LEADER
02:03:56.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:56.573536 28337 sys_catalog.cc:455] T 00000000000000000000000000000000 P 842b6c2a7322454eb92510275c056e25 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "89d379944adf49edb04c90a64ab032d0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "842b6c2a7322454eb92510275c056e25" member_type: VOTER last_known_addr { host: "127.27.102.254" port: 44423 } } peers { permanent_uuid: "89d379944adf49edb04c90a64ab032d0" member_type: VOTER last_known_addr { host: "127.27.102.253" port: 36957 } } peers { permanent_uuid: "2a9e163c188b436482ef90a861478b6e" member_type: VOTER last_known_addr { host: "127.27.102.252" port: 33421 } } }
02:03:56.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:56.574331 28337 sys_catalog.cc:458] T 00000000000000000000000000000000 P 842b6c2a7322454eb92510275c056e25 [sys.catalog]: This master's current role is: FOLLOWER
02:03:56.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:56.588075 28342 catalog_manager.cc:1382] Generated new cluster ID: 08893e88b5ad479c932e4f18c7ca5d19
02:03:56.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:56.588377 28342 catalog_manager.cc:1530] Initializing Kudu internal certificate authority...
02:03:56.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:56.589545 28356 catalog_manager.cc:1294] Loaded cluster ID: 08893e88b5ad479c932e4f18c7ca5d19
02:03:56.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:56.589813 28356 catalog_manager.cc:1587] T 00000000000000000000000000000000 P 842b6c2a7322454eb92510275c056e25: loading cluster ID for follower catalog manager: success
02:03:56.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:03:56.593763 28356 catalog_manager.cc:1611] T 00000000000000000000000000000000 P 842b6c2a7322454eb92510275c056e25: acquiring CA information for follower catalog manager: Not found: root CA entry not found
02:03:56.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:03:56.605680 28356 catalog_manager.cc:883] Not found: root CA entry not found: failed to prepare follower catalog manager, will retry
02:03:56.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:56.623788 28315 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2a9e163c188b436482ef90a861478b6e [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "89d379944adf49edb04c90a64ab032d0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "842b6c2a7322454eb92510275c056e25" member_type: VOTER last_known_addr { host: "127.27.102.254" port: 44423 } } peers { permanent_uuid: "89d379944adf49edb04c90a64ab032d0" member_type: VOTER last_known_addr { host: "127.27.102.253" port: 36957 } } peers { permanent_uuid: "2a9e163c188b436482ef90a861478b6e" member_type: VOTER last_known_addr { host: "127.27.102.252" port: 33421 } } }
02:03:56.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:56.624457 28315 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2a9e163c188b436482ef90a861478b6e [sys.catalog]: This master's current role is: FOLLOWER
02:03:56.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:56.664378 28342 catalog_manager.cc:1405] Generated new certificate authority record
02:03:56.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:56.666960 28342 catalog_manager.cc:1539] Loading token signing keys...
02:03:56.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:56.710773 28342 catalog_manager.cc:6069] T 00000000000000000000000000000000 P 89d379944adf49edb04c90a64ab032d0: Generated new TSK 0
02:03:56.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:56.712993 28342 catalog_manager.cc:1549] Initializing in-progress tserver states...
02:03:56.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:56.757388 28059 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskqQ5_Rr/build/tsan/bin/kudu
02:03:56.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskqQ5_Rr/build/tsan/bin/kudu
02:03:56.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster5755697716724601332/ts-0/wal
02:03:56.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster5755697716724601332/ts-0/data
02:03:56.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:03:56.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:03:56.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:03:56.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:03:56.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:03:56.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:03:56.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster5755697716724601332/ts-0/logs
02:03:56.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster5755697716724601332/ts-0/data/info.pb
02:03:56.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:03:56.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:03:56.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:03:56.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:03:56.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:03:56.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:03:56.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:03:56.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:03:56.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
02:03:56.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:03:56.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.102.193:0
02:03:56.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.27.102.193
02:03:56.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.102.193
02:03:56.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:03:56.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.27.102.254:44423,127.27.102.253:36957,127.27.102.252:33421
02:03:56.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.102.212:34925
02:03:56.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:03:56.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:03:56.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
02:03:56.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:56.932492 28332 catalog_manager.cc:1294] Loaded cluster ID: 08893e88b5ad479c932e4f18c7ca5d19
02:03:56.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:56.933233 28332 catalog_manager.cc:1587] T 00000000000000000000000000000000 P 2a9e163c188b436482ef90a861478b6e: loading cluster ID for follower catalog manager: success
02:03:56.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:56.941529 28332 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 2a9e163c188b436482ef90a861478b6e: acquiring CA information for follower catalog manager: success
02:03:56.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:56.969946 28332 catalog_manager.cc:1637] T 00000000000000000000000000000000 P 2a9e163c188b436482ef90a861478b6e: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
02:03:57.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:03:57.313911 28361 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
02:03:57.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:03:57.314564 28361 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:03:57.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:03:57.315301 28361 flags.cc:432] Enabled unsafe flag: --never_fsync=true
02:03:57.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:03:57.379106 28361 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
02:03:57.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:03:57.380501 28361 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.27.102.193
02:03:57.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:57.447691 28361 tablet_server_runner.cc:78] Tablet server non-default flags:
02:03:57.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:03:57.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.102.212:34925
02:03:57.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:03:57.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:03:57.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster5755697716724601332/ts-0/data
02:03:57.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster5755697716724601332/ts-0/wal
02:03:57.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:03:57.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:03:57.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.102.193:0
02:03:57.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:03:57.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:03:57.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:03:57.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster5755697716724601332/ts-0/data/info.pb
02:03:57.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.102.193
02:03:57.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:03:57.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.27.102.254:44423,127.27.102.253:36957,127.27.102.252:33421
02:03:57.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:03:57.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:03:57.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:03:57.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:03:57.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:03:57.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.27.102.193
02:03:57.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster5755697716724601332/ts-0/logs
02:03:57.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:03:57.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:03:57.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:03:57.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
02:03:57.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:03:57.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 23b10557f461bc2be3d2c282748d8de96f9c46ec
02:03:57.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:03:57.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 17 Jun 2026 01:43:23 UTC on e7f111948823
02:03:57.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12431
02:03:57.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:03:57.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:57.449448 28361 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:03:57.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:57.452119 28361 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:03:57.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:03:57.481767 28367 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:03:57.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:57.613690 28356 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 842b6c2a7322454eb92510275c056e25: acquiring CA information for follower catalog manager: success
02:03:57.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:57.618942 28356 catalog_manager.cc:1637] T 00000000000000000000000000000000 P 842b6c2a7322454eb92510275c056e25: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
02:03:59.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:03:58.876664 28366 debug-util.cc:398] Leaking SignalData structure 0x7b0800028000 after lost signal to thread 28361
02:03:59.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:03:59.791405 28361 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 2.307s user 0.594s sys 0.965s
02:03:59.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:03:59.791761 28361 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 2.307s user 0.594s sys 0.965s
02:03:59.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:03:57.486410 28368 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:03:59.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:59.803511 28361 server_base.cc:1061] running on GCE node
02:03:59.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:03:59.803514 28372 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:03:59.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:59.804808 28361 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:03:59.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:59.807089 28361 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:03:59.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:59.808452 28361 hybrid_clock.cc:648] HybridClock initialized: now 1781661839808410 us; error 43 us; skew 500 ppm
02:03:59.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:59.809294 28361 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:03:59.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:59.815864 28361 webserver.cc:533] Webserver started at http://127.27.102.193:34121/ using document root <none> and password file <none>
02:03:59.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:59.816784 28361 fs_manager.cc:362] Metadata directory not provided
02:03:59.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:59.817025 28361 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:03:59.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:59.817602 28361 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
02:03:59.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:59.821981 28361 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster5755697716724601332/ts-0/data/instance:
02:03:59.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6261622eea1d4016aba678255f6409cd"
02:03:59.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-17 02:03:59 on dist-test-slave-32ct"
02:03:59.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:59.823179 28361 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster5755697716724601332/ts-0/wal/instance:
02:03:59.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6261622eea1d4016aba678255f6409cd"
02:03:59.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-17 02:03:59 on dist-test-slave-32ct"
02:03:59.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:59.830741 28361 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.004s sys 0.004s
02:03:59.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:59.837138 28378 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:03:59.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:59.838414 28361 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.001s
02:03:59.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:59.838852 28361 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster5755697716724601332/ts-0/data,/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster5755697716724601332/ts-0/wal
02:03:59.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6261622eea1d4016aba678255f6409cd"
02:03:59.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-17 02:03:59 on dist-test-slave-32ct"
02:03:59.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:59.839296 28361 fs_report.cc:389] FS layout report
02:03:59.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:03:59.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster5755697716724601332/ts-0/wal
02:03:59.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster5755697716724601332/ts-0/wal
02:03:59.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster5755697716724601332/ts-0/data/data
02:03:59.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:03:59.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:03:59.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:03:59.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:03:59.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:03:59.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:03:59.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:03:59.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:03:59.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:03:59.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:03:59.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:03:59.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:59.897083 28361 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:03:59.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:59.898888 28361 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:03:59.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:59.899423 28361 kserver.cc:163] Server-wide thread pool size limit: 3276
02:03:59.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:59.902729 28361 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
02:03:59.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:59.906759 28361 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
02:03:59.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:59.906947 28361 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
02:03:59.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:59.907132 28361 ts_tablet_manager.cc:616] Registered 0 tablets
02:03:59.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:03:59.907260 28361 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
02:04:00.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:00.045297 28361 rpc_server.cc:307] RPC server started. Bound to: 127.27.102.193:35769
02:04:00.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:00.045475 28491 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.27.102.193:35769 every 8 connection(s)
02:04:00.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:00.048344 28361 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster5755697716724601332/ts-0/data/info.pb
02:04:00.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:00.048905 28059 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskqQ5_Rr/build/tsan/bin/kudu as pid 28361
02:04:00.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:00.049552 28059 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster5755697716724601332/ts-0/wal/instance
02:04:00.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:00.071990 28059 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskqQ5_Rr/build/tsan/bin/kudu
02:04:00.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskqQ5_Rr/build/tsan/bin/kudu
02:04:00.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster5755697716724601332/ts-1/wal
02:04:00.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster5755697716724601332/ts-1/data
02:04:00.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:04:00.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:04:00.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:04:00.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:04:00.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:04:00.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:04:00.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster5755697716724601332/ts-1/logs
02:04:00.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster5755697716724601332/ts-1/data/info.pb
02:04:00.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:04:00.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:04:00.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:04:00.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:04:00.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:04:00.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:04:00.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:04:00.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:04:00.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
02:04:00.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:04:00.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.102.194:0
02:04:00.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.27.102.194
02:04:00.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.102.194
02:04:00.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:04:00.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.27.102.254:44423,127.27.102.253:36957,127.27.102.252:33421
02:04:00.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.102.212:34925
02:04:00.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:04:00.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:04:00.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
02:04:00.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:00.094699 28493 heartbeater.cc:344] Connected to a master server at 127.27.102.253:36957
02:04:00.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:00.095232 28493 heartbeater.cc:461] Registering TS with master...
02:04:00.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:00.096706 28494 heartbeater.cc:344] Connected to a master server at 127.27.102.254:44423
02:04:00.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:00.096669 28493 heartbeater.cc:507] Master 127.27.102.253:36957 requested a full tablet report, sending...
02:04:00.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:00.097066 28494 heartbeater.cc:461] Registering TS with master...
02:04:00.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:00.097863 28494 heartbeater.cc:507] Master 127.27.102.254:44423 requested a full tablet report, sending...
02:04:00.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:00.098353 28492 heartbeater.cc:344] Connected to a master server at 127.27.102.252:33421
02:04:00.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:00.098735 28492 heartbeater.cc:461] Registering TS with master...
02:04:00.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:00.099704 28492 heartbeater.cc:507] Master 127.27.102.252:33421 requested a full tablet report, sending...
02:04:00.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:00.100910 28104 ts_manager.cc:194] Registered new tserver with Master: 6261622eea1d4016aba678255f6409cd (127.27.102.193:35769)
02:04:00.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:00.102391 28179 ts_manager.cc:194] Registered new tserver with Master: 6261622eea1d4016aba678255f6409cd (127.27.102.193:35769)
02:04:00.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:00.104969 28179 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.27.102.193:49863
02:04:00.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:00.107573 28260 ts_manager.cc:194] Registered new tserver with Master: 6261622eea1d4016aba678255f6409cd (127.27.102.193:35769)
02:04:00.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:00.390287 28502 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
02:04:00.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:00.390735 28502 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:04:00.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:00.391125 28502 flags.cc:432] Enabled unsafe flag: --never_fsync=true
02:04:00.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:00.423959 28502 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
02:04:00.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:00.424716 28502 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.27.102.194
02:04:00.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:00.461855 28502 tablet_server_runner.cc:78] Tablet server non-default flags:
02:04:00.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:04:00.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.102.212:34925
02:04:00.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:04:00.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:04:00.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster5755697716724601332/ts-1/data
02:04:00.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster5755697716724601332/ts-1/wal
02:04:00.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:04:00.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:04:00.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.102.194:0
02:04:00.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:04:00.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:04:00.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:04:00.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster5755697716724601332/ts-1/data/info.pb
02:04:00.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.102.194
02:04:00.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:04:00.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.27.102.254:44423,127.27.102.253:36957,127.27.102.252:33421
02:04:00.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:04:00.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:04:00.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:04:00.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:04:00.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:04:00.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.27.102.194
02:04:00.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster5755697716724601332/ts-1/logs
02:04:00.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:04:00.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:04:00.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:04:00.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
02:04:00.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:04:00.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 23b10557f461bc2be3d2c282748d8de96f9c46ec
02:04:00.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:04:00.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 17 Jun 2026 01:43:23 UTC on e7f111948823
02:04:00.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12431
02:04:00.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:04:00.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:00.463394 28502 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:04:00.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:00.464995 28502 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:04:00.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:00.476943 28508 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:04:01.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:01.111948 28493 heartbeater.cc:499] Master 127.27.102.253:36957 was elected leader, sending a full tablet report...
02:04:01.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:00.478549 28509 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:04:01.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:01.813788 28510 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1329 milliseconds
02:04:01.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:01.815387 28511 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:04:01.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:01.817824 28502 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.339s user 0.001s sys 0.009s
02:04:01.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:01.818173 28502 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.339s user 0.001s sys 0.009s
02:04:01.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:01.818465 28502 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:04:01.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:01.819995 28502 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:04:01.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:01.823173 28502 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:04:01.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:01.824615 28502 hybrid_clock.cc:648] HybridClock initialized: now 1781661841824531 us; error 93 us; skew 500 ppm
02:04:01.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:01.825851 28502 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:04:01.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:01.836673 28502 webserver.cc:533] Webserver started at http://127.27.102.194:43849/ using document root <none> and password file <none>
02:04:01.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:01.838335 28502 fs_manager.cc:362] Metadata directory not provided
02:04:01.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:01.838892 28502 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:04:01.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:01.839649 28502 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
02:04:01.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:01.848577 28502 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster5755697716724601332/ts-1/data/instance:
02:04:01.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8ab84846487b448da651d6973dc740e4"
02:04:01.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-17 02:04:01 on dist-test-slave-32ct"
02:04:01.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:01.850203 28502 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster5755697716724601332/ts-1/wal/instance:
02:04:01.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8ab84846487b448da651d6973dc740e4"
02:04:01.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-17 02:04:01 on dist-test-slave-32ct"
02:04:01.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:01.860147 28502 fs_manager.cc:696] Time spent creating directory manager: real 0.009s user 0.009s sys 0.000s
02:04:01.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:01.868439 28519 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:04:01.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:01.869837 28502 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.007s sys 0.000s
02:04:01.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:01.870292 28502 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster5755697716724601332/ts-1/data,/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster5755697716724601332/ts-1/wal
02:04:01.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8ab84846487b448da651d6973dc740e4"
02:04:01.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-17 02:04:01 on dist-test-slave-32ct"
02:04:01.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:01.870780 28502 fs_report.cc:389] FS layout report
02:04:01.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:04:01.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster5755697716724601332/ts-1/wal
02:04:01.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster5755697716724601332/ts-1/wal
02:04:01.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster5755697716724601332/ts-1/data/data
02:04:01.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:04:01.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:04:01.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:04:01.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:04:01.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:04:01.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:04:01.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:04:01.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:04:01.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:04:01.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:04:01.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:04:01.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:01.959184 28502 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:04:01.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:01.960611 28502 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:04:01.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:01.961026 28502 kserver.cc:163] Server-wide thread pool size limit: 3276
02:04:01.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:01.963624 28502 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
02:04:01.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:01.967677 28502 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
02:04:01.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:01.967941 28502 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
02:04:01.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:01.968397 28502 ts_tablet_manager.cc:616] Registered 0 tablets
02:04:01.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:01.968684 28502 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
02:04:02.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:02.105692 28502 rpc_server.cc:307] RPC server started. Bound to: 127.27.102.194:42945
02:04:02.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:02.105826 28631 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.27.102.194:42945 every 8 connection(s)
02:04:02.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:02.108564 28502 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster5755697716724601332/ts-1/data/info.pb
02:04:02.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:02.110821 28059 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskqQ5_Rr/build/tsan/bin/kudu as pid 28502
02:04:02.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:02.111308 28059 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster5755697716724601332/ts-1/wal/instance
02:04:02.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:02.139183 28059 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskqQ5_Rr/build/tsan/bin/kudu
02:04:02.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskqQ5_Rr/build/tsan/bin/kudu
02:04:02.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster5755697716724601332/ts-2/wal
02:04:02.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster5755697716724601332/ts-2/data
02:04:02.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:04:02.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:04:02.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:04:02.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:04:02.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:04:02.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:04:02.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster5755697716724601332/ts-2/logs
02:04:02.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster5755697716724601332/ts-2/data/info.pb
02:04:02.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:04:02.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:04:02.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:04:02.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:04:02.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:04:02.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:04:02.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:04:02.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:04:02.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
02:04:02.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:04:02.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.102.195:0
02:04:02.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.27.102.195
02:04:02.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.102.195
02:04:02.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:04:02.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.27.102.254:44423,127.27.102.253:36957,127.27.102.252:33421
02:04:02.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.102.212:34925
02:04:02.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:04:02.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:04:02.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
02:04:02.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:02.150331 28632 heartbeater.cc:344] Connected to a master server at 127.27.102.252:33421
02:04:02.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:02.150846 28632 heartbeater.cc:461] Registering TS with master...
02:04:02.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:02.152256 28632 heartbeater.cc:507] Master 127.27.102.252:33421 requested a full tablet report, sending...
02:04:02.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:02.155634 28260 ts_manager.cc:194] Registered new tserver with Master: 8ab84846487b448da651d6973dc740e4 (127.27.102.194:42945)
02:04:02.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:02.159436 28633 heartbeater.cc:344] Connected to a master server at 127.27.102.253:36957
02:04:02.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:02.159832 28633 heartbeater.cc:461] Registering TS with master...
02:04:02.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:02.160867 28633 heartbeater.cc:507] Master 127.27.102.253:36957 requested a full tablet report, sending...
02:04:02.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:02.163074 28634 heartbeater.cc:344] Connected to a master server at 127.27.102.254:44423
02:04:02.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:02.163420 28634 heartbeater.cc:461] Registering TS with master...
02:04:02.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:02.163432 28178 ts_manager.cc:194] Registered new tserver with Master: 8ab84846487b448da651d6973dc740e4 (127.27.102.194:42945)
02:04:02.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:02.164289 28634 heartbeater.cc:507] Master 127.27.102.254:44423 requested a full tablet report, sending...
02:04:02.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:02.165297 28178 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.27.102.194:51105
02:04:02.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:02.166988 28104 ts_manager.cc:194] Registered new tserver with Master: 8ab84846487b448da651d6973dc740e4 (127.27.102.194:42945)
02:04:02.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:02.454438 28642 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
02:04:02.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:02.454909 28642 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:04:02.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:02.455292 28642 flags.cc:432] Enabled unsafe flag: --never_fsync=true
02:04:02.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:02.488119 28642 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
02:04:02.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:02.488904 28642 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.27.102.195
02:04:02.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:02.525373 28642 tablet_server_runner.cc:78] Tablet server non-default flags:
02:04:02.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:04:02.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.102.212:34925
02:04:02.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:04:02.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:04:02.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster5755697716724601332/ts-2/data
02:04:02.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster5755697716724601332/ts-2/wal
02:04:02.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:04:02.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:04:02.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.102.195:0
02:04:02.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:04:02.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:04:02.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:04:02.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster5755697716724601332/ts-2/data/info.pb
02:04:02.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.102.195
02:04:02.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:04:02.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.27.102.254:44423,127.27.102.253:36957,127.27.102.252:33421
02:04:02.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:04:02.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:04:02.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:04:02.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:04:02.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:04:02.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.27.102.195
02:04:02.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster5755697716724601332/ts-2/logs
02:04:02.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:04:02.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:04:02.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:04:02.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
02:04:02.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:04:02.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 23b10557f461bc2be3d2c282748d8de96f9c46ec
02:04:02.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:04:02.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 17 Jun 2026 01:43:23 UTC on e7f111948823
02:04:02.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12431
02:04:02.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:04:02.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:02.526808 28642 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:04:02.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:02.528303 28642 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:04:02.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:02.540051 28648 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:04:03.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:03.168960 28633 heartbeater.cc:499] Master 127.27.102.253:36957 was elected leader, sending a full tablet report...
02:04:03.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:02.541849 28649 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:04:03.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:03.857442 28650 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
02:04:03.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:03.859982 28651 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:04:03.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:03.862545 28642 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.322s user 0.539s sys 0.782s
02:04:03.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:03.862928 28642 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.322s user 0.539s sys 0.782s
02:04:03.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:03.863291 28642 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:04:03.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:03.864671 28642 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:04:03.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:03.867241 28642 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:04:03.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:03.868611 28642 hybrid_clock.cc:648] HybridClock initialized: now 1781661843868624 us; error 87 us; skew 500 ppm
02:04:03.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:03.869587 28642 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:04:03.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:03.877146 28642 webserver.cc:533] Webserver started at http://127.27.102.195:42711/ using document root <none> and password file <none>
02:04:03.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:03.878222 28642 fs_manager.cc:362] Metadata directory not provided
02:04:03.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:03.878495 28642 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:04:03.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:03.879021 28642 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
02:04:03.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:03.883704 28642 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster5755697716724601332/ts-2/data/instance:
02:04:03.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e5fb453fb5524a14b21090d9fe0ce3e0"
02:04:03.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-17 02:04:03 on dist-test-slave-32ct"
02:04:03.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:03.884835 28642 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster5755697716724601332/ts-2/wal/instance:
02:04:03.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e5fb453fb5524a14b21090d9fe0ce3e0"
02:04:03.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-17 02:04:03 on dist-test-slave-32ct"
02:04:03.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:03.893582 28642 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.002s sys 0.005s
02:04:03.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:03.899655 28659 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:04:03.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:03.900811 28642 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.005s sys 0.001s
02:04:03.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:03.901152 28642 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster5755697716724601332/ts-2/data,/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster5755697716724601332/ts-2/wal
02:04:03.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e5fb453fb5524a14b21090d9fe0ce3e0"
02:04:03.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-17 02:04:03 on dist-test-slave-32ct"
02:04:03.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:03.901597 28642 fs_report.cc:389] FS layout report
02:04:03.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:04:03.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster5755697716724601332/ts-2/wal
02:04:03.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster5755697716724601332/ts-2/wal
02:04:03.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster5755697716724601332/ts-2/data/data
02:04:03.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:04:03.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:04:03.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:04:03.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:04:03.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:04:03.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:04:03.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:04:03.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:04:03.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:04:03.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:04:03.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:04:03.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:03.991811 28642 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:04:03.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:03.993398 28642 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:04:03.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:03.993916 28642 kserver.cc:163] Server-wide thread pool size limit: 3276
02:04:03.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:03.996694 28642 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
02:04:04.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:04.001063 28642 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
02:04:04.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:04.001286 28642 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
02:04:04.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:04.001533 28642 ts_tablet_manager.cc:616] Registered 0 tablets
02:04:04.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:04.001751 28642 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
02:04:04.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:04.146804 28642 rpc_server.cc:307] RPC server started. Bound to: 127.27.102.195:42917
02:04:04.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:04.146965 28771 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.27.102.195:42917 every 8 connection(s)
02:04:04.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:04.149785 28642 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster5755697716724601332/ts-2/data/info.pb
02:04:04.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:04.159144 28059 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskqQ5_Rr/build/tsan/bin/kudu as pid 28642
02:04:04.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:04.159729 28059 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster5755697716724601332/ts-2/wal/instance
02:04:04.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:04.189080 28773 heartbeater.cc:344] Connected to a master server at 127.27.102.253:36957
02:04:04.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:04.189540 28773 heartbeater.cc:461] Registering TS with master...
02:04:04.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:04.190917 28773 heartbeater.cc:507] Master 127.27.102.253:36957 requested a full tablet report, sending...
02:04:04.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:04.192798 28772 heartbeater.cc:344] Connected to a master server at 127.27.102.252:33421
02:04:04.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:04.193171 28772 heartbeater.cc:461] Registering TS with master...
02:04:04.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:04.193500 28179 ts_manager.cc:194] Registered new tserver with Master: e5fb453fb5524a14b21090d9fe0ce3e0 (127.27.102.195:42917)
02:04:04.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:04.194088 28772 heartbeater.cc:507] Master 127.27.102.252:33421 requested a full tablet report, sending...
02:04:04.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:04.195024 28179 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.27.102.195:35053
02:04:04.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:04.196913 28260 ts_manager.cc:194] Registered new tserver with Master: e5fb453fb5524a14b21090d9fe0ce3e0 (127.27.102.195:42917)
02:04:04.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:04.198356 28775 heartbeater.cc:344] Connected to a master server at 127.27.102.254:44423
02:04:04.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:04.198673 28775 heartbeater.cc:461] Registering TS with master...
02:04:04.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:04.199247 28775 heartbeater.cc:507] Master 127.27.102.254:44423 requested a full tablet report, sending...
02:04:04.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:04.201696 28104 ts_manager.cc:194] Registered new tserver with Master: e5fb453fb5524a14b21090d9fe0ce3e0 (127.27.102.195:42917)
02:04:04.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:04.212716 28059 external_mini_cluster.cc:955] 3 TS(s) registered with all masters
02:04:04.225 [DEBUG - main] (MiniKuduCluster.java:195) Response:
02:04:04.226 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
02:04:04.234 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.27.102.254"
port: 44423
}
bound_http_address {
host: "127.27.102.254"
port: 40841
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.27.102.253"
port: 36957
}
bound_http_address {
host: "127.27.102.253"
port: 33955
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.27.102.252"
port: 33421
}
bound_http_address {
host: "127.27.102.252"
port: 41293
}
}
}
02:04:04.236 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
02:04:04.238 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.27.102.193"
port: 35769
}
bound_http_address {
host: "127.27.102.193"
port: 34121
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.27.102.194"
port: 42945
}
bound_http_address {
host: "127.27.102.194"
port: 43849
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.27.102.195"
port: 42917
}
bound_http_address {
host: "127.27.102.195"
port: 42711
}
}
}
02:04:04.238 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
02:04:04.397 [DEBUG - main] (CompositeStrategy.java:59) Unable to load modern Subject methods, relying only on legacy methods
java.lang.NoSuchMethodException: javax.security.auth.Subject.current()
at java.base/java.lang.Class.getDeclaredMethod(Class.java:2677)
at org.apache.kudu.client.internals.ModernStrategy.<init>(ModernStrategy.java:51)
at org.apache.kudu.client.internals.CompositeStrategy.<init>(CompositeStrategy.java:52)
at org.apache.kudu.client.internals.CompositeStrategy.<clinit>(CompositeStrategy.java:41)
at org.apache.kudu.client.internals.SecurityManagerCompatibility.get(SecurityManagerCompatibility.java:43)
at org.apache.kudu.client.SecurityContext.setupSubject(SecurityContext.java:155)
at org.apache.kudu.client.SecurityContext.<init>(SecurityContext.java:138)
at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:431)
at org.apache.kudu.client.AsyncKuduClient$AsyncKuduClientBuilder.build(AsyncKuduClient.java:3109)
at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:161)
at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runners.Suite.runChild(Suite.java:128)
at org.junit.runners.Suite.runChild(Suite.java:27)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runner.JUnitCore.run(JUnitCore.java:137)
at org.junit.runner.JUnitCore.run(JUnitCore.java:115)
at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77)
at org.junit.runner.JUnitCore.main(JUnitCore.java:36)
java.lang.NoSuchMethodException: javax.security.auth.Subject.current()
at java.base/java.lang.Class.getDeclaredMethod(Class.java:2677)
at org.apache.kudu.client.internals.ModernStrategy.<init>(ModernStrategy.java:51)
at org.apache.kudu.client.internals.CompositeStrategy.<init>(CompositeStrategy.java:52)
at org.apache.kudu.client.internals.CompositeStrategy.<clinit>(CompositeStrategy.java:41)
at org.apache.kudu.client.internals.SecurityManagerCompatibility.get(SecurityManagerCompatibility.java:43)
at org.apache.kudu.client.SecurityContext.setupSubject(SecurityContext.java:155)
at org.apache.kudu.client.SecurityContext.<init>(SecurityContext.java:138)
at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:431)
at org.apache.kudu.client.AsyncKuduClient$AsyncKuduClientBuilder.build(AsyncKuduClient.java:3109)
at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:161)
at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runners.Suite.runChild(Suite.java:128)
at org.junit.runners.Suite.runChild(Suite.java:27)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runner.JUnitCore.run(JUnitCore.java:137)
at org.junit.runner.JUnitCore.run(JUnitCore.java:115)
at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77)
at org.junit.runner.JUnitCore.main(JUnitCore.java:36)
02:04:04.443 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
02:04:04.550 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskqQ5_Rr/test-tmp
02:04:04.550 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
02:04:04.551 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskqQ5_Rr/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
02:04:04.554 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957"
extra_master_flags: "--webserver_require_spnego=false"
extra_master_flags: "--authn_token_validity_seconds=10"
extra_tserver_flags: "--webserver_require_spnego=false"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
02:04:04.911 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:04:04.911 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
02:04:04.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:04.916363 28784 mini_oidc.cc:150] Starting JWKS server
02:04:04.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:04.919445 28784 webserver.cc:533] Webserver started at http://127.0.0.1:35823/ using document root <none> and password file <none>
02:04:04.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:04.919878 28784 mini_oidc.cc:186] Starting OIDC Discovery server
02:04:04.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:04.921649 28784 webserver.cc:533] Webserver started at http://127.0.0.1:44397/ using document root <none> and password file <none>
02:04:04.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
02:04:04.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/krb5kdc/principal' for realm 'KRBTEST.COM',
02:04:05.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
02:04:05.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 17 02:04:05 dist-test-slave-32ct krb5kdc[28797](info): setting up network...
02:04:05.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
02:04:05.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 17 02:04:05 dist-test-slave-32ct krb5kdc[28797](info): set up 2 sockets
02:04:05.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 17 02:04:05 dist-test-slave-32ct krb5kdc[28797](info): commencing operation
02:04:05.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
02:04:05.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:05.199038 28773 heartbeater.cc:499] Master 127.27.102.253:36957 was elected leader, sending a full tablet report...
02:04:07.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:07.062562 28784 mini_kdc.cc:121] Time spent starting KDC: real 2.103s user 0.004s sys 0.016s
02:04:07.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
02:04:07.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
02:04:07.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
02:04:07.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
02:04:07.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
02:04:07.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
02:04:07.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
02:04:07.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
02:04:07.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
02:04:07.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
02:04:07.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/krb5kdc/test-user.keytab.
02:04:07.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/krb5kdc/test-user.keytab.
02:04:07.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 17 02:04:07 dist-test-slave-32ct krb5kdc[28797](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781661847, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:04:07.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
02:04:07.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-17T02:04:07Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
02:04:07.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-17T02:04:07Z Disabled control of system clock
02:04:07.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.28.28.62@KRBTEST.COM; defaulting to no policy
02:04:07.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:04:07.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.28.28.62@KRBTEST.COM" created.
02:04:07.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:04:07.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.28.28.62 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/krb5kdc/kudu.keytab.
02:04:07.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.28.28.62 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/krb5kdc/kudu.keytab.
02:04:07.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.28.28.62@KRBTEST.COM; defaulting to no policy
02:04:07.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:04:07.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.28.28.62@KRBTEST.COM" created.
02:04:07.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:04:07.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.28.28.62 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/krb5kdc/kudu.keytab.
02:04:07.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.28.28.62 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/krb5kdc/kudu.keytab.
02:04:07.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:07.290726 28784 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskqQ5_Rr/build/tsan/bin/kudu
02:04:07.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskqQ5_Rr/build/tsan/bin/kudu
02:04:07.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/master-0/wal
02:04:07.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/master-0/data
02:04:07.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:04:07.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:04:07.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:04:07.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:04:07.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:04:07.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:04:07.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/master-0/logs
02:04:07.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/master-0/data/info.pb
02:04:07.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:04:07.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:04:07.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:04:07.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:04:07.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:04:07.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:04:07.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:04:07.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:04:07.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:04:07.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:04:07.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:04:07.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:04:07.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.28.62:46845
02:04:07.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.28.62
02:04:07.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:04:07.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/krb5kdc/kudu.keytab
02:04:07.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.28.62
02:04:07.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:04:07.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
02:04:07.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
02:04:07.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
02:04:07.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.28.20:40741
02:04:07.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:04:07.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:04:07.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:04:07.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:04:07.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.28.62:46845,127.28.28.61:38085,127.28.28.60:37309
02:04:07.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:04:07.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35823/jwks.json
02:04:07.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
02:04:07.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
02:04:07.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:07.601023 28816 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
02:04:07.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:07.601701 28816 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:04:07.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:07.602272 28816 flags.cc:432] Enabled unsafe flag: --never_fsync=true
02:04:07.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:07.636693 28816 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
02:04:07.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:07.637007 28816 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
02:04:07.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:07.637287 28816 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
02:04:07.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:07.637550 28816 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
02:04:07.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:07.637765 28816 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
02:04:07.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:07.637956 28816 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
02:04:07.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:07.638104 28816 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:35823/jwks.json
02:04:07.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:07.677452 28816 master_runner.cc:387] Master server non-default flags:
02:04:07.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:04:07.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.28.20:40741
02:04:07.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:04:07.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:04:07.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/master-0/data
02:04:07.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/master-0/wal
02:04:07.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
02:04:07.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:04:07.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.28.62:46845,127.28.28.61:38085,127.28.28.60:37309
02:04:07.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/krb5kdc/kudu.keytab
02:04:07.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.28.62
02:04:07.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:04:07.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:04:07.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:04:07.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.28.62:46845
02:04:07.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:04:07.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:04:07.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:04:07.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35823/jwks.json
02:04:07.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:04:07.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
02:04:07.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
02:04:07.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:04:07.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:04:07.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/master-0/data/info.pb
02:04:07.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.28.62
02:04:07.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:04:07.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:04:07.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:04:07.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:04:07.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:04:07.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:04:07.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/master-0/logs
02:04:07.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:04:07.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:04:07.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:04:07.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:04:07.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:04:07.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 23b10557f461bc2be3d2c282748d8de96f9c46ec
02:04:07.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:04:07.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 17 Jun 2026 01:43:23 UTC on e7f111948823
02:04:07.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12431
02:04:07.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:04:07.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:07.678834 28816 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:04:07.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:07.679911 28816 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:04:07.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:07.694258 28822 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:04:09.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:07.696107 28823 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:04:09.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:09.031364 28816 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.336s user 0.001s sys 0.003s
02:04:09.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:09.031422 28824 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1330 milliseconds
02:04:09.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:09.032107 28816 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.337s user 0.003s sys 0.004s
02:04:09.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:09.033250 28825 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:04:09.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:09.033314 28816 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:04:09.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:09.034626 28816 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:04:09.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:09.037214 28816 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:04:09.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:09.038643 28816 hybrid_clock.cc:648] HybridClock initialized: now 1781661849038603 us; error 48 us; skew 500 ppm
02:04:09.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:09.039516 28816 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:04:09.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 17 02:04:09 dist-test-slave-32ct krb5kdc[28797](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781661849, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.28.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:04:09.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:09.046892 28816 init.cc:377] Logged in from keytab as kudu/127.28.28.62@KRBTEST.COM (short username kudu)
02:04:09.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:09.056934 28816 webserver.cc:533] Webserver started at http://127.28.28.62:35423/ using document root <none> and password file <none>
02:04:09.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:09.057968 28816 fs_manager.cc:362] Metadata directory not provided
02:04:09.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:09.058228 28816 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:04:09.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:09.058842 28816 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
02:04:09.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:09.063230 28816 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/master-0/data/instance:
02:04:09.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4ce4fbc4c4344da38c81053f87cf6862"
02:04:09.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-17 02:04:09 on dist-test-slave-32ct"
02:04:09.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:09.064467 28816 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/master-0/wal/instance:
02:04:09.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4ce4fbc4c4344da38c81053f87cf6862"
02:04:09.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-17 02:04:09 on dist-test-slave-32ct"
02:04:09.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:09.073024 28816 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.006s sys 0.005s
02:04:09.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:09.079136 28833 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:04:09.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:09.080312 28816 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
02:04:09.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:09.080590 28816 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/master-0/data,/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/master-0/wal
02:04:09.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4ce4fbc4c4344da38c81053f87cf6862"
02:04:09.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-17 02:04:09 on dist-test-slave-32ct"
02:04:09.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:09.080883 28816 fs_report.cc:389] FS layout report
02:04:09.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:04:09.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/master-0/wal
02:04:09.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/master-0/wal
02:04:09.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/master-0/data/data
02:04:09.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:04:09.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:04:09.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:04:09.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:04:09.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:04:09.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:04:09.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:04:09.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:04:09.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:04:09.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:04:09.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:04:09.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:09.167627 28816 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:04:09.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:09.173368 28816 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:04:09.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:09.173758 28816 kserver.cc:163] Server-wide thread pool size limit: 3276
02:04:09.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:09.252249 28816 rpc_server.cc:307] RPC server started. Bound to: 127.28.28.62:46845
02:04:09.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:09.252425 28885 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.28.28.62:46845 every 8 connection(s)
02:04:09.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:09.255052 28816 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/master-0/data/info.pb
02:04:09.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:09.261145 28886 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
02:04:09.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:09.263422 28784 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskqQ5_Rr/build/tsan/bin/kudu as pid 28816
02:04:09.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:09.263844 28784 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/master-0/wal/instance
02:04:09.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:09.273118 28886 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.28.62" port: 46845 } has no permanent_uuid. Determining permanent_uuid...
02:04:09.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.28.28.61@KRBTEST.COM; defaulting to no policy
02:04:09.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:04:09.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.28.28.61@KRBTEST.COM" created.
02:04:09.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 17 02:04:09 dist-test-slave-32ct krb5kdc[28797](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781661849, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.28.62@KRBTEST.COM for kudu/127.28.28.62@KRBTEST.COM
02:04:09.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:04:09.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.28.28.61 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/krb5kdc/kudu.keytab.
02:04:09.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.28.28.61 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/krb5kdc/kudu.keytab.
02:04:09.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:09.316918 28886 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.28.61" port: 38085 } has no permanent_uuid. Determining permanent_uuid...
02:04:09.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:09.319473 28838 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.28.28.61:38085: connect: Connection refused (error 111)
02:04:09.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.28.28.61@KRBTEST.COM; defaulting to no policy
02:04:09.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:09.323647 28886 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.28.61:38085: Network error: Client connection negotiation failed: client connection to 127.28.28.61:38085: connect: Connection refused (error 111)
02:04:09.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:04:09.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.28.28.61@KRBTEST.COM" created.
02:04:09.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:04:09.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.28.28.61 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/krb5kdc/kudu.keytab.
02:04:09.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.28.28.61 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/krb5kdc/kudu.keytab.
02:04:09.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:09.348037 28784 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskqQ5_Rr/build/tsan/bin/kudu
02:04:09.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskqQ5_Rr/build/tsan/bin/kudu
02:04:09.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/master-1/wal
02:04:09.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/master-1/data
02:04:09.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:04:09.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:04:09.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:04:09.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:04:09.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:04:09.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:04:09.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/master-1/logs
02:04:09.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/master-1/data/info.pb
02:04:09.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:04:09.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:04:09.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:04:09.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:04:09.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:04:09.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:04:09.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:04:09.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:04:09.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:04:09.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:04:09.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:04:09.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:04:09.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.28.61:38085
02:04:09.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.28.61
02:04:09.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:04:09.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/krb5kdc/kudu.keytab
02:04:09.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.28.61
02:04:09.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:04:09.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
02:04:09.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
02:04:09.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
02:04:09.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.28.20:40741
02:04:09.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:04:09.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:04:09.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:04:09.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:04:09.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.28.62:46845,127.28.28.61:38085,127.28.28.60:37309
02:04:09.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:04:09.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35823/jwks.json
02:04:09.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
02:04:09.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
02:04:09.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:09.373185 28886 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.28.61" port: 38085 } attempt: 1
02:04:09.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:09.377415 28886 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.28.61:38085: Network error: Client connection negotiation failed: client connection to 127.28.28.61:38085: connect: Connection refused (error 111)
02:04:09.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:09.445850 28886 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.28.61" port: 38085 } attempt: 2
02:04:09.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:09.450062 28886 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.28.61:38085: Network error: Client connection negotiation failed: client connection to 127.28.28.61:38085: connect: Connection refused (error 111)
02:04:09.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:09.541538 28886 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.28.61" port: 38085 } attempt: 3
02:04:09.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:09.545944 28886 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.28.61:38085: Network error: Client connection negotiation failed: client connection to 127.28.28.61:38085: connect: Connection refused (error 111)
02:04:09.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:09.663690 28895 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
02:04:09.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:09.664269 28895 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:04:09.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:09.664819 28895 flags.cc:432] Enabled unsafe flag: --never_fsync=true
02:04:09.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:09.689566 28886 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.28.61" port: 38085 } attempt: 4
02:04:09.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:09.694355 28886 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.28.61:38085: Network error: Client connection negotiation failed: client connection to 127.28.28.61:38085: connect: Connection refused (error 111)
02:04:09.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:09.700642 28895 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
02:04:09.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:09.700954 28895 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
02:04:09.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:09.701517 28895 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
02:04:09.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:09.701839 28895 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
02:04:09.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:09.702188 28895 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
02:04:09.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:09.702517 28895 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
02:04:09.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:09.702755 28895 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:35823/jwks.json
02:04:09.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:09.741823 28895 master_runner.cc:387] Master server non-default flags:
02:04:09.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:04:09.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.28.20:40741
02:04:09.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:04:09.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:04:09.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/master-1/data
02:04:09.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/master-1/wal
02:04:09.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
02:04:09.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:04:09.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.28.62:46845,127.28.28.61:38085,127.28.28.60:37309
02:04:09.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/krb5kdc/kudu.keytab
02:04:09.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.28.61
02:04:09.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:04:09.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:04:09.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:04:09.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.28.61:38085
02:04:09.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:04:09.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:04:09.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:04:09.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35823/jwks.json
02:04:09.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:04:09.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
02:04:09.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
02:04:09.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:04:09.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:04:09.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/master-1/data/info.pb
02:04:09.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.28.61
02:04:09.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:04:09.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:04:09.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:04:09.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:04:09.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:04:09.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:04:09.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/master-1/logs
02:04:09.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:04:09.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:04:09.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:04:09.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:04:09.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:04:09.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 23b10557f461bc2be3d2c282748d8de96f9c46ec
02:04:09.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:04:09.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 17 Jun 2026 01:43:23 UTC on e7f111948823
02:04:09.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12431
02:04:09.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:04:09.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:09.743453 28895 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:04:09.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:09.744913 28895 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:04:09.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:09.755983 28902 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:04:09.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:09.993777 28886 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.28.61" port: 38085 } attempt: 5
02:04:10.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:10.003814 28886 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.28.61:38085: Network error: Client connection negotiation failed: client connection to 127.28.28.61:38085: connect: Connection refused (error 111)
02:04:10.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:10.551317 28886 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.28.61" port: 38085 } attempt: 6
02:04:10.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:10.581866 28886 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.28.61:38085: Network error: Client connection negotiation failed: client connection to 127.28.28.61:38085: connect: Connection refused (error 111)
02:04:11.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:11.160634 28901 debug-util.cc:398] Leaking SignalData structure 0x7b080002ef00 after lost signal to thread 28895
02:04:11.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:11.524614 28901 kernel_stack_watchdog.cc:198] Thread 28895 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 401ms:
02:04:11.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
02:04:11.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
02:04:11.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
02:04:11.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
02:04:11.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:09.757863 28903 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:04:11.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:11.525705 28895 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.769s user 0.646s sys 1.086s
02:04:11.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:11.526398 28895 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.770s user 0.647s sys 1.087s
02:04:11.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:11.527889 28905 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:04:11.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:11.530936 28904 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection timed out after 1774 milliseconds
02:04:11.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:11.530977 28895 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:04:11.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:11.532469 28895 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:04:11.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:11.535336 28895 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:04:11.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:11.536949 28895 hybrid_clock.cc:648] HybridClock initialized: now 1781661851536909 us; error 41 us; skew 500 ppm
02:04:11.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:11.538023 28895 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:04:11.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 17 02:04:11 dist-test-slave-32ct krb5kdc[28797](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781661851, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.28.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:04:11.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:11.545194 28895 init.cc:377] Logged in from keytab as kudu/127.28.28.61@KRBTEST.COM (short username kudu)
02:04:11.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:11.550622 28895 webserver.cc:533] Webserver started at http://127.28.28.61:34599/ using document root <none> and password file <none>
02:04:11.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:11.551592 28895 fs_manager.cc:362] Metadata directory not provided
02:04:11.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:11.551852 28895 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:04:11.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:11.552380 28895 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
02:04:11.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:11.556635 28895 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/master-1/data/instance:
02:04:11.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2c6224a4528c494eb782e1c4e5a71288"
02:04:11.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-17 02:04:11 on dist-test-slave-32ct"
02:04:11.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:11.557871 28895 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/master-1/wal/instance:
02:04:11.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2c6224a4528c494eb782e1c4e5a71288"
02:04:11.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-17 02:04:11 on dist-test-slave-32ct"
02:04:11.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:11.564940 28895 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.006s sys 0.000s
02:04:11.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:11.570433 28917 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:04:11.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:11.571427 28895 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.003s sys 0.002s
02:04:11.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:11.571787 28895 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/master-1/data,/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/master-1/wal
02:04:11.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2c6224a4528c494eb782e1c4e5a71288"
02:04:11.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-17 02:04:11 on dist-test-slave-32ct"
02:04:11.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:11.572131 28895 fs_report.cc:389] FS layout report
02:04:11.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:04:11.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/master-1/wal
02:04:11.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/master-1/wal
02:04:11.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/master-1/data/data
02:04:11.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:04:11.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:04:11.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:04:11.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:04:11.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:04:11.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:04:11.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:04:11.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:04:11.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:04:11.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:04:11.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:04:11.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:11.624799 28895 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:04:11.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:11.635483 28895 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:04:11.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:11.636021 28895 kserver.cc:163] Server-wide thread pool size limit: 3276
02:04:11.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:11.642400 28886 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.28.61" port: 38085 } attempt: 7
02:04:11.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:11.649151 28886 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.28.61:38085: Network error: Client connection negotiation failed: client connection to 127.28.28.61:38085: connect: Connection refused (error 111)
02:04:11.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:11.712177 28895 rpc_server.cc:307] RPC server started. Bound to: 127.28.28.61:38085
02:04:11.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:11.712287 28970 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.28.28.61:38085 every 8 connection(s)
02:04:11.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:11.714958 28895 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/master-1/data/info.pb
02:04:11.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:11.720656 28784 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskqQ5_Rr/build/tsan/bin/kudu as pid 28895
02:04:11.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:11.721091 28784 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/master-1/wal/instance
02:04:11.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:11.721269 28971 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
02:04:11.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:11.732110 28971 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.28.62" port: 46845 } has no permanent_uuid. Determining permanent_uuid...
02:04:11.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.28.28.60@KRBTEST.COM; defaulting to no policy
02:04:11.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:04:11.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.28.28.60@KRBTEST.COM" created.
02:04:11.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 17 02:04:11 dist-test-slave-32ct krb5kdc[28797](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781661851, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.28.61@KRBTEST.COM for kudu/127.28.28.62@KRBTEST.COM
02:04:11.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:04:11.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.28.28.60 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/krb5kdc/kudu.keytab.
02:04:11.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.28.28.60 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/krb5kdc/kudu.keytab.
02:04:11.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:11.774302 28971 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.28.61" port: 38085 } has no permanent_uuid. Determining permanent_uuid...
02:04:11.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.28.28.60@KRBTEST.COM; defaulting to no policy
02:04:11.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:04:11.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.28.28.60@KRBTEST.COM" created.
02:04:11.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 17 02:04:11 dist-test-slave-32ct krb5kdc[28797](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781661851, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.28.61@KRBTEST.COM for kudu/127.28.28.61@KRBTEST.COM
02:04:11.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:11.799201 28971 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.28.60" port: 37309 } has no permanent_uuid. Determining permanent_uuid...
02:04:11.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:11.801774 28918 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.28.28.60:37309: connect: Connection refused (error 111)
02:04:11.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:04:11.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.28.28.60 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/krb5kdc/kudu.keytab.
02:04:11.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.28.28.60 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/krb5kdc/kudu.keytab.
02:04:11.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:11.806129 28971 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.28.60:37309: Network error: Client connection negotiation failed: client connection to 127.28.28.60:37309: connect: Connection refused (error 111)
02:04:11.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:11.806326 28784 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskqQ5_Rr/build/tsan/bin/kudu
02:04:11.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskqQ5_Rr/build/tsan/bin/kudu
02:04:11.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/master-2/wal
02:04:11.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/master-2/data
02:04:11.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:04:11.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:04:11.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:04:11.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:04:11.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:04:11.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:04:11.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/master-2/logs
02:04:11.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/master-2/data/info.pb
02:04:11.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:04:11.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:04:11.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:04:11.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:04:11.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:04:11.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:04:11.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:04:11.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:04:11.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:04:11.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:04:11.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:04:11.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:04:11.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.28.60:37309
02:04:11.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.28.60
02:04:11.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:04:11.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/krb5kdc/kudu.keytab
02:04:11.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.28.60
02:04:11.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:04:11.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
02:04:11.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
02:04:11.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
02:04:11.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.28.20:40741
02:04:11.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:04:11.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:04:11.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:04:11.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:04:11.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.28.62:46845,127.28.28.61:38085,127.28.28.60:37309
02:04:11.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:04:11.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35823/jwks.json
02:04:11.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
02:04:11.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
02:04:11.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:11.855700 28971 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.28.60" port: 37309 } attempt: 1
02:04:11.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:11.860770 28971 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.28.60:37309: Network error: Client connection negotiation failed: client connection to 127.28.28.60:37309: connect: Connection refused (error 111)
02:04:11.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:11.929176 28971 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.28.60" port: 37309 } attempt: 2
02:04:11.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:11.933905 28971 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.28.60:37309: Network error: Client connection negotiation failed: client connection to 127.28.28.60:37309: connect: Connection refused (error 111)
02:04:12.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:12.025302 28971 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.28.60" port: 37309 } attempt: 3
02:04:12.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:12.030635 28971 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.28.60:37309: Network error: Client connection negotiation failed: client connection to 127.28.28.60:37309: connect: Connection refused (error 111)
02:04:12.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:12.145138 28981 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
02:04:12.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:12.145736 28981 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:04:12.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:12.146283 28981 flags.cc:432] Enabled unsafe flag: --never_fsync=true
02:04:12.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:12.174108 28971 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.28.60" port: 37309 } attempt: 4
02:04:12.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:12.178362 28971 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.28.60:37309: Network error: Client connection negotiation failed: client connection to 127.28.28.60:37309: connect: Connection refused (error 111)
02:04:12.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:12.181500 28981 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
02:04:12.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:12.181720 28981 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
02:04:12.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:12.181986 28981 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
02:04:12.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:12.182201 28981 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
02:04:12.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:12.182395 28981 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
02:04:12.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:12.182611 28981 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
02:04:12.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:12.182861 28981 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:35823/jwks.json
02:04:12.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:12.230863 28981 master_runner.cc:387] Master server non-default flags:
02:04:12.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:04:12.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.28.20:40741
02:04:12.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:04:12.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:04:12.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/master-2/data
02:04:12.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/master-2/wal
02:04:12.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
02:04:12.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:04:12.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.28.62:46845,127.28.28.61:38085,127.28.28.60:37309
02:04:12.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/krb5kdc/kudu.keytab
02:04:12.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.28.60
02:04:12.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:04:12.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:04:12.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:04:12.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.28.60:37309
02:04:12.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:04:12.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:04:12.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:04:12.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35823/jwks.json
02:04:12.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:04:12.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
02:04:12.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
02:04:12.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:04:12.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:04:12.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/master-2/data/info.pb
02:04:12.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.28.60
02:04:12.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:04:12.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:04:12.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:04:12.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:04:12.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:04:12.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:04:12.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/master-2/logs
02:04:12.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:04:12.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:04:12.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:04:12.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:04:12.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:04:12.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 23b10557f461bc2be3d2c282748d8de96f9c46ec
02:04:12.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:04:12.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 17 Jun 2026 01:43:23 UTC on e7f111948823
02:04:12.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12431
02:04:12.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:04:12.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:12.232156 28981 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:04:12.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:12.233330 28981 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:04:12.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:12.243707 28988 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:04:12.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:12.477799 28971 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.28.60" port: 37309 } attempt: 5
02:04:12.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:12.487545 28971 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.28.60:37309: Network error: Client connection negotiation failed: client connection to 127.28.28.60:37309: connect: Connection refused (error 111)
02:04:13.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:13.035177 28971 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.28.60" port: 37309 } attempt: 6
02:04:13.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:13.072271 28971 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.28.60:37309: Network error: Client connection negotiation failed: client connection to 127.28.28.60:37309: connect: Connection refused (error 111)
02:04:13.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:13.739634 28886 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.28.61" port: 38085 } attempt: 8
02:04:13.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 17 02:04:13 dist-test-slave-32ct krb5kdc[28797](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781661849, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.28.62@KRBTEST.COM for kudu/127.28.28.61@KRBTEST.COM
02:04:13.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:13.825829 28886 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.28.60" port: 37309 } has no permanent_uuid. Determining permanent_uuid...
02:04:13.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:13.835311 28886 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.28.60:37309: Network error: Client connection negotiation failed: client connection to 127.28.28.60:37309: connect: Connection refused (error 111)
02:04:13.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:13.648627 28987 debug-util.cc:398] Leaking SignalData structure 0x7b080002ef00 after lost signal to thread 28981
02:04:13.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:13.856192 28987 kernel_stack_watchdog.cc:198] Thread 28981 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 400ms:
02:04:13.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
02:04:13.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
02:04:13.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
02:04:13.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
02:04:13.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:12.245657 28989 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:04:13.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:13.858515 28981 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.614s user 0.509s sys 0.986s
02:04:13.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:13.859005 28981 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.614s user 0.509s sys 0.986s
02:04:13.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:13.859052 28991 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:04:13.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:13.863286 28990 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1614 milliseconds
02:04:13.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:13.863291 28981 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:04:13.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:13.864619 28981 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:04:13.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:13.867015 28981 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:04:13.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:13.868407 28981 hybrid_clock.cc:648] HybridClock initialized: now 1781661853868373 us; error 37 us; skew 500 ppm
02:04:13.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:13.869215 28981 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:04:13.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 17 02:04:13 dist-test-slave-32ct krb5kdc[28797](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781661853, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.28.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:04:13.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:13.875274 28981 init.cc:377] Logged in from keytab as kudu/127.28.28.60@KRBTEST.COM (short username kudu)
02:04:13.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:13.880051 28981 webserver.cc:533] Webserver started at http://127.28.28.60:38177/ using document root <none> and password file <none>
02:04:13.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:13.880918 28981 fs_manager.cc:362] Metadata directory not provided
02:04:13.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:13.881098 28981 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:04:13.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:13.881537 28981 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
02:04:13.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:13.885707 28981 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/master-2/data/instance:
02:04:13.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "698282ccaf55491badb00732fef7a918"
02:04:13.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-17 02:04:13 on dist-test-slave-32ct"
02:04:13.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:13.886744 28981 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/master-2/wal/instance:
02:04:13.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "698282ccaf55491badb00732fef7a918"
02:04:13.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-17 02:04:13 on dist-test-slave-32ct"
02:04:13.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:13.893857 28981 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.005s sys 0.002s
02:04:13.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:13.898918 29006 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:04:13.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:13.899927 28981 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.005s sys 0.001s
02:04:13.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:13.900187 28981 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/master-2/data,/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/master-2/wal
02:04:13.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "698282ccaf55491badb00732fef7a918"
02:04:13.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-17 02:04:13 on dist-test-slave-32ct"
02:04:13.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:13.900457 28981 fs_report.cc:389] FS layout report
02:04:13.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:04:13.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/master-2/wal
02:04:13.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/master-2/wal
02:04:13.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/master-2/data/data
02:04:13.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:04:13.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:04:13.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:04:13.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:04:13.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:04:13.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:04:13.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:04:13.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:04:13.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:04:13.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:04:13.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:04:13.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:13.901902 28886 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.28.60" port: 37309 } attempt: 1
02:04:13.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:13.906553 28886 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.28.60:37309: Network error: Client connection negotiation failed: client connection to 127.28.28.60:37309: connect: Connection refused (error 111)
02:04:13.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:13.958252 28981 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:04:13.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:13.959975 28886 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.28.60" port: 37309 } attempt: 2
02:04:13.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:13.964672 28886 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.28.60:37309: Network error: Client connection negotiation failed: client connection to 127.28.28.60:37309: connect: Connection refused (error 111)
02:04:13.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:13.965483 28981 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:04:13.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:13.966029 28981 kserver.cc:163] Server-wide thread pool size limit: 3276
02:04:14.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:14.040653 28981 rpc_server.cc:307] RPC server started. Bound to: 127.28.28.60:37309
02:04:14.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:14.040704 29057 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.28.28.60:37309 every 8 connection(s)
02:04:14.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:14.041078 28886 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.28.60" port: 37309 } attempt: 3
02:04:14.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:14.044574 28981 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/master-2/data/info.pb
02:04:14.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:14.051222 28784 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskqQ5_Rr/build/tsan/bin/kudu as pid 28981
02:04:14.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:14.051815 28784 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/master-2/wal/instance
02:04:14.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:14.058223 29059 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
02:04:14.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 17 02:04:14 dist-test-slave-32ct krb5kdc[28797](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781661849, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.28.62@KRBTEST.COM for kudu/127.28.28.60@KRBTEST.COM
02:04:14.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:14.076618 29059 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.28.62" port: 46845 } has no permanent_uuid. Determining permanent_uuid...
02:04:14.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 17 02:04:14 dist-test-slave-32ct krb5kdc[28797](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781661847, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.28.28.62@KRBTEST.COM
02:04:14.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:14.104914 28886 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4ce4fbc4c4344da38c81053f87cf6862: Bootstrap starting.
02:04:14.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 17 02:04:14 dist-test-slave-32ct krb5kdc[28797](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781661853, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.28.60@KRBTEST.COM for kudu/127.28.28.62@KRBTEST.COM
02:04:14.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:14.113955 28886 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 4ce4fbc4c4344da38c81053f87cf6862: Neither blocks nor log segments found. Creating new log.
02:04:14.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:14.116536 28886 log.cc:826] T 00000000000000000000000000000000 P 4ce4fbc4c4344da38c81053f87cf6862: Log is configured to *not* fsync() on all Append() calls
02:04:14.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:14.124058 29059 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.28.61" port: 38085 } has no permanent_uuid. Determining permanent_uuid...
02:04:14.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:14.130860 28886 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4ce4fbc4c4344da38c81053f87cf6862: No bootstrap required, opened a new log
02:04:14.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:14.132858 28971 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.28.60" port: 37309 } attempt: 7
02:04:14.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 17 02:04:14 dist-test-slave-32ct krb5kdc[28797](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781661847, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.28.28.61@KRBTEST.COM
02:04:14.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 17 02:04:14 dist-test-slave-32ct krb5kdc[28797](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781661853, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.28.60@KRBTEST.COM for kudu/127.28.28.61@KRBTEST.COM
02:04:14.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:14.154000 29059 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.28.60" port: 37309 } has no permanent_uuid. Determining permanent_uuid...
02:04:14.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 17 02:04:14 dist-test-slave-32ct krb5kdc[28797](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781661851, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.28.61@KRBTEST.COM for kudu/127.28.28.60@KRBTEST.COM
02:04:14.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 17 02:04:14 dist-test-slave-32ct krb5kdc[28797](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781661847, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.28.28.60@KRBTEST.COM
02:04:14.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 17 02:04:14 dist-test-slave-32ct krb5kdc[28797](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781661853, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.28.60@KRBTEST.COM for kudu/127.28.28.60@KRBTEST.COM
02:04:14.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:14.179023 28886 raft_consensus.cc:359] T 00000000000000000000000000000000 P 4ce4fbc4c4344da38c81053f87cf6862 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4ce4fbc4c4344da38c81053f87cf6862" member_type: VOTER last_known_addr { host: "127.28.28.62" port: 46845 } } peers { permanent_uuid: "2c6224a4528c494eb782e1c4e5a71288" member_type: VOTER last_known_addr { host: "127.28.28.61" port: 38085 } } peers { permanent_uuid: "698282ccaf55491badb00732fef7a918" member_type: VOTER last_known_addr { host: "127.28.28.60" port: 37309 } }
02:04:14.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:14.180217 28886 raft_consensus.cc:385] T 00000000000000000000000000000000 P 4ce4fbc4c4344da38c81053f87cf6862 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:04:14.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:14.180698 28886 raft_consensus.cc:740] T 00000000000000000000000000000000 P 4ce4fbc4c4344da38c81053f87cf6862 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4ce4fbc4c4344da38c81053f87cf6862, State: Initialized, Role: FOLLOWER
02:04:14.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:14.181994 28886 consensus_queue.cc:260] T 00000000000000000000000000000000 P 4ce4fbc4c4344da38c81053f87cf6862 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4ce4fbc4c4344da38c81053f87cf6862" member_type: VOTER last_known_addr { host: "127.28.28.62" port: 46845 } } peers { permanent_uuid: "2c6224a4528c494eb782e1c4e5a71288" member_type: VOTER last_known_addr { host: "127.28.28.61" port: 38085 } } peers { permanent_uuid: "698282ccaf55491badb00732fef7a918" member_type: VOTER last_known_addr { host: "127.28.28.60" port: 37309 } }
02:04:14.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:14.192216 29070 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4ce4fbc4c4344da38c81053f87cf6862 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4ce4fbc4c4344da38c81053f87cf6862" member_type: VOTER last_known_addr { host: "127.28.28.62" port: 46845 } } peers { permanent_uuid: "2c6224a4528c494eb782e1c4e5a71288" member_type: VOTER last_known_addr { host: "127.28.28.61" port: 38085 } } peers { permanent_uuid: "698282ccaf55491badb00732fef7a918" member_type: VOTER last_known_addr { host: "127.28.28.60" port: 37309 } } }
02:04:14.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:14.193328 29070 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4ce4fbc4c4344da38c81053f87cf6862 [sys.catalog]: This master's current role is: FOLLOWER
02:04:14.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:14.194146 28886 sys_catalog.cc:565] T 00000000000000000000000000000000 P 4ce4fbc4c4344da38c81053f87cf6862 [sys.catalog]: configured and running, proceeding with master startup.
02:04:14.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:14.196343 28971 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 2c6224a4528c494eb782e1c4e5a71288: Bootstrap starting.
02:04:14.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:14.198527 29059 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 698282ccaf55491badb00732fef7a918: Bootstrap starting.
02:04:14.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:14.207118 29059 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 698282ccaf55491badb00732fef7a918: Neither blocks nor log segments found. Creating new log.
02:04:14.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:14.209363 29059 log.cc:826] T 00000000000000000000000000000000 P 698282ccaf55491badb00732fef7a918: Log is configured to *not* fsync() on all Append() calls
02:04:14.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:14.210733 28971 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 2c6224a4528c494eb782e1c4e5a71288: Neither blocks nor log segments found. Creating new log.
02:04:14.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:14.214113 28971 log.cc:826] T 00000000000000000000000000000000 P 2c6224a4528c494eb782e1c4e5a71288: Log is configured to *not* fsync() on all Append() calls
02:04:14.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:14.218146 29059 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 698282ccaf55491badb00732fef7a918: No bootstrap required, opened a new log
02:04:14.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:14.219798 28971 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 2c6224a4528c494eb782e1c4e5a71288: No bootstrap required, opened a new log
02:04:14.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:14.247671 29059 raft_consensus.cc:359] T 00000000000000000000000000000000 P 698282ccaf55491badb00732fef7a918 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4ce4fbc4c4344da38c81053f87cf6862" member_type: VOTER last_known_addr { host: "127.28.28.62" port: 46845 } } peers { permanent_uuid: "2c6224a4528c494eb782e1c4e5a71288" member_type: VOTER last_known_addr { host: "127.28.28.61" port: 38085 } } peers { permanent_uuid: "698282ccaf55491badb00732fef7a918" member_type: VOTER last_known_addr { host: "127.28.28.60" port: 37309 } }
02:04:14.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:14.248560 29059 raft_consensus.cc:385] T 00000000000000000000000000000000 P 698282ccaf55491badb00732fef7a918 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:04:14.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:14.248891 29059 raft_consensus.cc:740] T 00000000000000000000000000000000 P 698282ccaf55491badb00732fef7a918 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 698282ccaf55491badb00732fef7a918, State: Initialized, Role: FOLLOWER
02:04:14.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:14.259966 29059 consensus_queue.cc:260] T 00000000000000000000000000000000 P 698282ccaf55491badb00732fef7a918 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4ce4fbc4c4344da38c81053f87cf6862" member_type: VOTER last_known_addr { host: "127.28.28.62" port: 46845 } } peers { permanent_uuid: "2c6224a4528c494eb782e1c4e5a71288" member_type: VOTER last_known_addr { host: "127.28.28.61" port: 38085 } } peers { permanent_uuid: "698282ccaf55491badb00732fef7a918" member_type: VOTER last_known_addr { host: "127.28.28.60" port: 37309 } }
02:04:14.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:14.265829 29083 sys_catalog.cc:455] T 00000000000000000000000000000000 P 698282ccaf55491badb00732fef7a918 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4ce4fbc4c4344da38c81053f87cf6862" member_type: VOTER last_known_addr { host: "127.28.28.62" port: 46845 } } peers { permanent_uuid: "2c6224a4528c494eb782e1c4e5a71288" member_type: VOTER last_known_addr { host: "127.28.28.61" port: 38085 } } peers { permanent_uuid: "698282ccaf55491badb00732fef7a918" member_type: VOTER last_known_addr { host: "127.28.28.60" port: 37309 } } }
02:04:14.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:14.266611 29083 sys_catalog.cc:458] T 00000000000000000000000000000000 P 698282ccaf55491badb00732fef7a918 [sys.catalog]: This master's current role is: FOLLOWER
02:04:14.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:14.266851 28971 raft_consensus.cc:359] T 00000000000000000000000000000000 P 2c6224a4528c494eb782e1c4e5a71288 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4ce4fbc4c4344da38c81053f87cf6862" member_type: VOTER last_known_addr { host: "127.28.28.62" port: 46845 } } peers { permanent_uuid: "2c6224a4528c494eb782e1c4e5a71288" member_type: VOTER last_known_addr { host: "127.28.28.61" port: 38085 } } peers { permanent_uuid: "698282ccaf55491badb00732fef7a918" member_type: VOTER last_known_addr { host: "127.28.28.60" port: 37309 } }
02:04:14.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:14.267976 28971 raft_consensus.cc:385] T 00000000000000000000000000000000 P 2c6224a4528c494eb782e1c4e5a71288 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:04:14.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:14.268731 28971 raft_consensus.cc:740] T 00000000000000000000000000000000 P 2c6224a4528c494eb782e1c4e5a71288 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 2c6224a4528c494eb782e1c4e5a71288, State: Initialized, Role: FOLLOWER
02:04:14.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:14.270661 28971 consensus_queue.cc:260] T 00000000000000000000000000000000 P 2c6224a4528c494eb782e1c4e5a71288 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4ce4fbc4c4344da38c81053f87cf6862" member_type: VOTER last_known_addr { host: "127.28.28.62" port: 46845 } } peers { permanent_uuid: "2c6224a4528c494eb782e1c4e5a71288" member_type: VOTER last_known_addr { host: "127.28.28.61" port: 38085 } } peers { permanent_uuid: "698282ccaf55491badb00732fef7a918" member_type: VOTER last_known_addr { host: "127.28.28.60" port: 37309 } }
02:04:14.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:14.273738 29059 sys_catalog.cc:565] T 00000000000000000000000000000000 P 698282ccaf55491badb00732fef7a918 [sys.catalog]: configured and running, proceeding with master startup.
02:04:14.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:14.285084 29070 raft_consensus.cc:493] T 00000000000000000000000000000000 P 4ce4fbc4c4344da38c81053f87cf6862 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:04:14.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:14.285522 29086 catalog_manager.cc:805] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
02:04:14.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:14.285630 29070 raft_consensus.cc:515] T 00000000000000000000000000000000 P 4ce4fbc4c4344da38c81053f87cf6862 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4ce4fbc4c4344da38c81053f87cf6862" member_type: VOTER last_known_addr { host: "127.28.28.62" port: 46845 } } peers { permanent_uuid: "2c6224a4528c494eb782e1c4e5a71288" member_type: VOTER last_known_addr { host: "127.28.28.61" port: 38085 } } peers { permanent_uuid: "698282ccaf55491badb00732fef7a918" member_type: VOTER last_known_addr { host: "127.28.28.60" port: 37309 } }
02:04:14.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:14.290114 29033 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "4ce4fbc4c4344da38c81053f87cf6862" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "698282ccaf55491badb00732fef7a918" is_pre_election: true
02:04:14.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:14.291051 29033 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 698282ccaf55491badb00732fef7a918 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 4ce4fbc4c4344da38c81053f87cf6862 in term 0.
02:04:14.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:14.291465 29084 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2c6224a4528c494eb782e1c4e5a71288 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4ce4fbc4c4344da38c81053f87cf6862" member_type: VOTER last_known_addr { host: "127.28.28.62" port: 46845 } } peers { permanent_uuid: "2c6224a4528c494eb782e1c4e5a71288" member_type: VOTER last_known_addr { host: "127.28.28.61" port: 38085 } } peers { permanent_uuid: "698282ccaf55491badb00732fef7a918" member_type: VOTER last_known_addr { host: "127.28.28.60" port: 37309 } } }
02:04:14.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:14.292407 29084 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2c6224a4528c494eb782e1c4e5a71288 [sys.catalog]: This master's current role is: FOLLOWER
02:04:14.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:14.292850 28971 sys_catalog.cc:565] T 00000000000000000000000000000000 P 2c6224a4528c494eb782e1c4e5a71288 [sys.catalog]: configured and running, proceeding with master startup.
02:04:14.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:14.293531 28835 leader_election.cc:304] T 00000000000000000000000000000000 P 4ce4fbc4c4344da38c81053f87cf6862 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4ce4fbc4c4344da38c81053f87cf6862, 698282ccaf55491badb00732fef7a918; no voters:
02:04:14.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:14.294307 29070 leader_election.cc:290] T 00000000000000000000000000000000 P 4ce4fbc4c4344da38c81053f87cf6862 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 2c6224a4528c494eb782e1c4e5a71288 (127.28.28.61:38085), 698282ccaf55491badb00732fef7a918 (127.28.28.60:37309)
02:04:14.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:14.294839 29070 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 4ce4fbc4c4344da38c81053f87cf6862 [term 0 FOLLOWER]: Leader pre-election won for term 1
02:04:14.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:14.295260 29070 raft_consensus.cc:493] T 00000000000000000000000000000000 P 4ce4fbc4c4344da38c81053f87cf6862 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
02:04:14.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:14.295238 28946 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "4ce4fbc4c4344da38c81053f87cf6862" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "2c6224a4528c494eb782e1c4e5a71288" is_pre_election: true
02:04:14.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:14.295606 29070 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 4ce4fbc4c4344da38c81053f87cf6862 [term 0 FOLLOWER]: Advancing to term 1
02:04:14.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:14.296038 28946 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 2c6224a4528c494eb782e1c4e5a71288 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 4ce4fbc4c4344da38c81053f87cf6862 in term 0.
02:04:14.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:14.307220 29070 raft_consensus.cc:515] T 00000000000000000000000000000000 P 4ce4fbc4c4344da38c81053f87cf6862 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4ce4fbc4c4344da38c81053f87cf6862" member_type: VOTER last_known_addr { host: "127.28.28.62" port: 46845 } } peers { permanent_uuid: "2c6224a4528c494eb782e1c4e5a71288" member_type: VOTER last_known_addr { host: "127.28.28.61" port: 38085 } } peers { permanent_uuid: "698282ccaf55491badb00732fef7a918" member_type: VOTER last_known_addr { host: "127.28.28.60" port: 37309 } }
02:04:14.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:14.314949 29070 leader_election.cc:290] T 00000000000000000000000000000000 P 4ce4fbc4c4344da38c81053f87cf6862 [CANDIDATE]: Term 1 election: Requested vote from peers 2c6224a4528c494eb782e1c4e5a71288 (127.28.28.61:38085), 698282ccaf55491badb00732fef7a918 (127.28.28.60:37309)
02:04:14.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:14.317691 29033 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "4ce4fbc4c4344da38c81053f87cf6862" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "698282ccaf55491badb00732fef7a918"
02:04:14.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:14.318403 29033 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 698282ccaf55491badb00732fef7a918 [term 0 FOLLOWER]: Advancing to term 1
02:04:14.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:14.321700 28946 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "4ce4fbc4c4344da38c81053f87cf6862" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "2c6224a4528c494eb782e1c4e5a71288"
02:04:14.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:14.322410 28946 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 2c6224a4528c494eb782e1c4e5a71288 [term 0 FOLLOWER]: Advancing to term 1
02:04:14.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:14.332403 28946 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 2c6224a4528c494eb782e1c4e5a71288 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 4ce4fbc4c4344da38c81053f87cf6862 in term 1.
02:04:14.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:14.335319 29033 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 698282ccaf55491badb00732fef7a918 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 4ce4fbc4c4344da38c81053f87cf6862 in term 1.
02:04:14.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:14.339751 28835 leader_election.cc:304] T 00000000000000000000000000000000 P 4ce4fbc4c4344da38c81053f87cf6862 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4ce4fbc4c4344da38c81053f87cf6862, 698282ccaf55491badb00732fef7a918; no voters:
02:04:14.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:14.340755 29070 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 4ce4fbc4c4344da38c81053f87cf6862 [term 1 FOLLOWER]: Leader election won for term 1
02:04:14.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:14.341270 29070 raft_consensus.cc:697] T 00000000000000000000000000000000 P 4ce4fbc4c4344da38c81053f87cf6862 [term 1 LEADER]: Becoming Leader. State: Replica: 4ce4fbc4c4344da38c81053f87cf6862, State: Running, Role: LEADER
02:04:14.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:14.342442 29070 consensus_queue.cc:237] T 00000000000000000000000000000000 P 4ce4fbc4c4344da38c81053f87cf6862 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4ce4fbc4c4344da38c81053f87cf6862" member_type: VOTER last_known_addr { host: "127.28.28.62" port: 46845 } } peers { permanent_uuid: "2c6224a4528c494eb782e1c4e5a71288" member_type: VOTER last_known_addr { host: "127.28.28.61" port: 38085 } } peers { permanent_uuid: "698282ccaf55491badb00732fef7a918" member_type: VOTER last_known_addr { host: "127.28.28.60" port: 37309 } }
02:04:14.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:14.365989 29087 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4ce4fbc4c4344da38c81053f87cf6862 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 4ce4fbc4c4344da38c81053f87cf6862. Latest consensus state: current_term: 1 leader_uuid: "4ce4fbc4c4344da38c81053f87cf6862" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4ce4fbc4c4344da38c81053f87cf6862" member_type: VOTER last_known_addr { host: "127.28.28.62" port: 46845 } } peers { permanent_uuid: "2c6224a4528c494eb782e1c4e5a71288" member_type: VOTER last_known_addr { host: "127.28.28.61" port: 38085 } } peers { permanent_uuid: "698282ccaf55491badb00732fef7a918" member_type: VOTER last_known_addr { host: "127.28.28.60" port: 37309 } } }
02:04:14.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:14.366791 29087 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4ce4fbc4c4344da38c81053f87cf6862 [sys.catalog]: This master's current role is: LEADER
02:04:14.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:14.335433 28816 master_runner.cc:187] Error getting master registration for 127.28.28.61:38085: OK, instance_id { permanent_uuid: "2c6224a4528c494eb782e1c4e5a71288" instance_seqno: 1781661851691887 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
02:04:14.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:14.381196 28816 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
02:04:14.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:14.384258 29100 catalog_manager.cc:1510] Loading table and tablet metadata into memory...
02:04:14.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:14.397254 28923 tablet.cc:2406] T 00000000000000000000000000000000 P 2c6224a4528c494eb782e1c4e5a71288: Can't schedule compaction. Clean time has not been advanced past its initial value.
02:04:14.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:14.398924 29100 catalog_manager.cc:1519] Initializing Kudu cluster ID...
02:04:14.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:14.438169 29107 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 2c6224a4528c494eb782e1c4e5a71288: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:04:14.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:14.438527 29107 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:04:14.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:14.438692 28840 tablet.cc:2406] T 00000000000000000000000000000000 P 4ce4fbc4c4344da38c81053f87cf6862: Can't schedule compaction. Clean time has not been advanced past its initial value.
02:04:14.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:14.440496 29113 catalog_manager.cc:805] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
02:04:14.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:14.471881 29012 tablet.cc:2406] T 00000000000000000000000000000000 P 698282ccaf55491badb00732fef7a918: Can't schedule compaction. Clean time has not been advanced past its initial value.
02:04:14.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:14.473026 29033 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 698282ccaf55491badb00732fef7a918 [term 1 FOLLOWER]: Refusing update from remote peer 4ce4fbc4c4344da38c81053f87cf6862: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
02:04:14.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:14.431797 28895 master_runner.cc:187] Error getting master registration for 127.28.28.60:37309: OK, instance_id { permanent_uuid: "698282ccaf55491badb00732fef7a918" instance_seqno: 1781661854021149 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
02:04:14.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:14.474159 28895 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
02:04:14.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:14.475842 28946 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 2c6224a4528c494eb782e1c4e5a71288 [term 1 FOLLOWER]: Refusing update from remote peer 4ce4fbc4c4344da38c81053f87cf6862: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
02:04:14.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:14.475584 29087 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 4ce4fbc4c4344da38c81053f87cf6862 [LEADER]: Connected to new peer: Peer: permanent_uuid: "698282ccaf55491badb00732fef7a918" member_type: VOTER last_known_addr { host: "127.28.28.60" port: 37309 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
02:04:14.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:14.477524 29087 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 4ce4fbc4c4344da38c81053f87cf6862 [LEADER]: Connected to new peer: Peer: permanent_uuid: "2c6224a4528c494eb782e1c4e5a71288" member_type: VOTER last_known_addr { host: "127.28.28.61" port: 38085 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
02:04:14.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:14.512161 29083 sys_catalog.cc:455] T 00000000000000000000000000000000 P 698282ccaf55491badb00732fef7a918 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 4ce4fbc4c4344da38c81053f87cf6862. Latest consensus state: current_term: 1 leader_uuid: "4ce4fbc4c4344da38c81053f87cf6862" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4ce4fbc4c4344da38c81053f87cf6862" member_type: VOTER last_known_addr { host: "127.28.28.62" port: 46845 } } peers { permanent_uuid: "2c6224a4528c494eb782e1c4e5a71288" member_type: VOTER last_known_addr { host: "127.28.28.61" port: 38085 } } peers { permanent_uuid: "698282ccaf55491badb00732fef7a918" member_type: VOTER last_known_addr { host: "127.28.28.60" port: 37309 } } }
02:04:14.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:14.512918 29083 sys_catalog.cc:458] T 00000000000000000000000000000000 P 698282ccaf55491badb00732fef7a918 [sys.catalog]: This master's current role is: FOLLOWER
02:04:14.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:14.520877 29111 mvcc.cc:204] Tried to move back new op lower bound from 7297686955904229376 to 7297686955417219072. Current Snapshot: MvccSnapshot[applied={T|T < 7297686955904229376}]
02:04:14.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:14.534420 29070 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4ce4fbc4c4344da38c81053f87cf6862 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "4ce4fbc4c4344da38c81053f87cf6862" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4ce4fbc4c4344da38c81053f87cf6862" member_type: VOTER last_known_addr { host: "127.28.28.62" port: 46845 } } peers { permanent_uuid: "2c6224a4528c494eb782e1c4e5a71288" member_type: VOTER last_known_addr { host: "127.28.28.61" port: 38085 } } peers { permanent_uuid: "698282ccaf55491badb00732fef7a918" member_type: VOTER last_known_addr { host: "127.28.28.60" port: 37309 } } }
02:04:14.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:14.537588 29100 catalog_manager.cc:1382] Generated new cluster ID: f94c5ca418a8467e9c05f9f73c940f49
02:04:14.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:14.537693 29070 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4ce4fbc4c4344da38c81053f87cf6862 [sys.catalog]: This master's current role is: LEADER
02:04:14.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:14.537891 29100 catalog_manager.cc:1530] Initializing Kudu internal certificate authority...
02:04:14.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:14.538421 29070 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4ce4fbc4c4344da38c81053f87cf6862 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "4ce4fbc4c4344da38c81053f87cf6862" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4ce4fbc4c4344da38c81053f87cf6862" member_type: VOTER last_known_addr { host: "127.28.28.62" port: 46845 } } peers { permanent_uuid: "2c6224a4528c494eb782e1c4e5a71288" member_type: VOTER last_known_addr { host: "127.28.28.61" port: 38085 } } peers { permanent_uuid: "698282ccaf55491badb00732fef7a918" member_type: VOTER last_known_addr { host: "127.28.28.60" port: 37309 } } }
02:04:14.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:14.539258 29070 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4ce4fbc4c4344da38c81053f87cf6862 [sys.catalog]: This master's current role is: LEADER
02:04:14.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:14.531682 29084 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2c6224a4528c494eb782e1c4e5a71288 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 4ce4fbc4c4344da38c81053f87cf6862. Latest consensus state: current_term: 1 leader_uuid: "4ce4fbc4c4344da38c81053f87cf6862" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4ce4fbc4c4344da38c81053f87cf6862" member_type: VOTER last_known_addr { host: "127.28.28.62" port: 46845 } } peers { permanent_uuid: "2c6224a4528c494eb782e1c4e5a71288" member_type: VOTER last_known_addr { host: "127.28.28.61" port: 38085 } } peers { permanent_uuid: "698282ccaf55491badb00732fef7a918" member_type: VOTER last_known_addr { host: "127.28.28.60" port: 37309 } } }
02:04:14.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:14.540944 29084 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2c6224a4528c494eb782e1c4e5a71288 [sys.catalog]: This master's current role is: FOLLOWER
02:04:14.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:14.548018 29115 mvcc.cc:204] Tried to move back new op lower bound from 7297686955904229376 to 7297686955417219072. Current Snapshot: MvccSnapshot[applied={T|T < 7297686955904229376}]
02:04:14.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:14.562508 29084 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2c6224a4528c494eb782e1c4e5a71288 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "4ce4fbc4c4344da38c81053f87cf6862" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4ce4fbc4c4344da38c81053f87cf6862" member_type: VOTER last_known_addr { host: "127.28.28.62" port: 46845 } } peers { permanent_uuid: "2c6224a4528c494eb782e1c4e5a71288" member_type: VOTER last_known_addr { host: "127.28.28.61" port: 38085 } } peers { permanent_uuid: "698282ccaf55491badb00732fef7a918" member_type: VOTER last_known_addr { host: "127.28.28.60" port: 37309 } } }
02:04:14.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:14.563422 29084 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2c6224a4528c494eb782e1c4e5a71288 [sys.catalog]: This master's current role is: FOLLOWER
02:04:14.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:14.571830 29083 sys_catalog.cc:455] T 00000000000000000000000000000000 P 698282ccaf55491badb00732fef7a918 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "4ce4fbc4c4344da38c81053f87cf6862" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4ce4fbc4c4344da38c81053f87cf6862" member_type: VOTER last_known_addr { host: "127.28.28.62" port: 46845 } } peers { permanent_uuid: "2c6224a4528c494eb782e1c4e5a71288" member_type: VOTER last_known_addr { host: "127.28.28.61" port: 38085 } } peers { permanent_uuid: "698282ccaf55491badb00732fef7a918" member_type: VOTER last_known_addr { host: "127.28.28.60" port: 37309 } } }
02:04:14.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:14.572566 29083 sys_catalog.cc:458] T 00000000000000000000000000000000 P 698282ccaf55491badb00732fef7a918 [sys.catalog]: This master's current role is: FOLLOWER
02:04:14.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:14.608510 29100 catalog_manager.cc:1405] Generated new certificate authority record
02:04:14.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:14.610960 29100 catalog_manager.cc:1539] Loading token signing keys...
02:04:14.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:14.640730 29100 catalog_manager.cc:6069] T 00000000000000000000000000000000 P 4ce4fbc4c4344da38c81053f87cf6862: Generated new TSK 0
02:04:14.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:14.642385 29100 catalog_manager.cc:1549] Initializing in-progress tserver states...
02:04:14.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.28.28.1@KRBTEST.COM; defaulting to no policy
02:04:14.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:04:14.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.28.28.1@KRBTEST.COM" created.
02:04:14.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:04:14.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.28.28.1 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/krb5kdc/kudu.keytab.
02:04:14.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.28.28.1 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/krb5kdc/kudu.keytab.
02:04:14.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.28.28.1@KRBTEST.COM; defaulting to no policy
02:04:14.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:04:14.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.28.28.1@KRBTEST.COM" created.
02:04:14.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:04:14.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.28.28.1 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/krb5kdc/kudu.keytab.
02:04:14.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.28.28.1 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/krb5kdc/kudu.keytab.
02:04:14.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:14.775820 28784 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskqQ5_Rr/build/tsan/bin/kudu
02:04:14.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskqQ5_Rr/build/tsan/bin/kudu
02:04:14.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/ts-0/wal
02:04:14.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/ts-0/data
02:04:14.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:04:14.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:04:14.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:04:14.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:04:14.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:04:14.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:04:14.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/ts-0/logs
02:04:14.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/ts-0/data/info.pb
02:04:14.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:04:14.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:04:14.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:04:14.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:04:14.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:04:14.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:04:14.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:04:14.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:04:14.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
02:04:14.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:04:14.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.28.1:0
02:04:14.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.28.28.1
02:04:14.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.28.1
02:04:14.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:04:14.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.28.28.62:46845,127.28.28.61:38085,127.28.28.60:37309
02:04:14.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/krb5kdc/kudu.keytab
02:04:14.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.28.1
02:04:14.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:04:14.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
02:04:14.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
02:04:14.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
02:04:14.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.28.20:40741
02:04:14.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:04:14.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:04:14.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:04:14.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false with env {KRB5CCNAME=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
02:04:15.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:15.256845 29126 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
02:04:15.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:15.258729 29126 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:04:15.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:15.259877 29126 flags.cc:432] Enabled unsafe flag: --never_fsync=true
02:04:15.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:15.327757 29126 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
02:04:15.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:15.329311 29126 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.28.28.1
02:04:15.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:15.396538 29126 tablet_server_runner.cc:78] Tablet server non-default flags:
02:04:15.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:04:15.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.28.20:40741
02:04:15.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:04:15.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:04:15.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/ts-0/data
02:04:15.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/ts-0/wal
02:04:15.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/krb5kdc/kudu.keytab
02:04:15.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.28.1
02:04:15.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:04:15.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:04:15.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.28.1:0
02:04:15.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:04:15.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:04:15.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
02:04:15.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
02:04:15.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:04:15.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:04:15.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/ts-0/data/info.pb
02:04:15.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.28.1
02:04:15.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:04:15.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.28.28.62:46845,127.28.28.61:38085,127.28.28.60:37309
02:04:15.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:04:15.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:04:15.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:04:15.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:04:15.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:04:15.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.28.28.1
02:04:15.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/ts-0/logs
02:04:15.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:04:15.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:04:15.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:04:15.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
02:04:15.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:04:15.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 23b10557f461bc2be3d2c282748d8de96f9c46ec
02:04:15.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:04:15.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 17 Jun 2026 01:43:23 UTC on e7f111948823
02:04:15.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12431
02:04:15.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:04:15.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:15.398262 29126 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:04:15.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:15.399574 29126 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:04:15.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:15.414882 29133 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:04:15.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:15.450795 29107 catalog_manager.cc:1294] Loaded cluster ID: f94c5ca418a8467e9c05f9f73c940f49
02:04:15.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:15.451557 29107 catalog_manager.cc:1587] T 00000000000000000000000000000000 P 2c6224a4528c494eb782e1c4e5a71288: loading cluster ID for follower catalog manager: success
02:04:15.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:15.479103 29113 catalog_manager.cc:1294] Loaded cluster ID: f94c5ca418a8467e9c05f9f73c940f49
02:04:15.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:15.484421 29107 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 2c6224a4528c494eb782e1c4e5a71288: acquiring CA information for follower catalog manager: success
02:04:15.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:15.485180 29113 catalog_manager.cc:1587] T 00000000000000000000000000000000 P 698282ccaf55491badb00732fef7a918: loading cluster ID for follower catalog manager: success
02:04:15.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:15.528854 29107 catalog_manager.cc:1637] T 00000000000000000000000000000000 P 2c6224a4528c494eb782e1c4e5a71288: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
02:04:15.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:15.536531 29113 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 698282ccaf55491badb00732fef7a918: acquiring CA information for follower catalog manager: success
02:04:15.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:15.575657 29113 catalog_manager.cc:1637] T 00000000000000000000000000000000 P 698282ccaf55491badb00732fef7a918: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
02:04:18.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:18.552807 29135 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 3134 milliseconds
02:04:18.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:18.553037 29126 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 3.123s user 0.001s sys 0.000s
02:04:18.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:18.566393 29126 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 3.137s user 0.002s sys 0.000s
02:04:18.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:18.566629 29126 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:04:18.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:16.815939 29132 debug-util.cc:398] Leaking SignalData structure 0x7b08000370e0 after lost signal to thread 29126
02:04:18.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:15.417687 29134 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:04:18.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:18.555088 29137 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:04:18.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:18.575505 29126 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:04:18.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:18.585520 29126 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:04:18.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:18.589361 29126 hybrid_clock.cc:648] HybridClock initialized: now 1781661858589214 us; error 130 us; skew 500 ppm
02:04:18.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:18.595369 29126 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:04:18.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 17 02:04:18 dist-test-slave-32ct krb5kdc[28797](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781661858, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.28.1@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:04:18.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:18.603856 29126 init.cc:377] Logged in from keytab as kudu/127.28.28.1@KRBTEST.COM (short username kudu)
02:04:18.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:18.634440 29126 webserver.cc:533] Webserver started at http://127.28.28.1:45263/ using document root <none> and password file <none>
02:04:18.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:18.636037 29126 fs_manager.cc:362] Metadata directory not provided
02:04:18.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:18.636565 29126 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:04:18.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:18.637285 29126 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
02:04:18.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:18.643784 29126 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/ts-0/data/instance:
02:04:18.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "232880c2803f4de0851f2c9578cf2128"
02:04:18.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-17 02:04:18 on dist-test-slave-32ct"
02:04:18.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:18.645597 29126 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/ts-0/wal/instance:
02:04:18.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "232880c2803f4de0851f2c9578cf2128"
02:04:18.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-17 02:04:18 on dist-test-slave-32ct"
02:04:18.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:18.660897 29126 fs_manager.cc:696] Time spent creating directory manager: real 0.014s user 0.005s sys 0.008s
02:04:18.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:18.674263 29149 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:04:18.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:18.675832 29126 fs_manager.cc:730] Time spent opening block manager: real 0.010s user 0.003s sys 0.005s
02:04:18.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:18.676412 29126 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/ts-0/data,/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/ts-0/wal
02:04:18.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "232880c2803f4de0851f2c9578cf2128"
02:04:18.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-17 02:04:18 on dist-test-slave-32ct"
02:04:18.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:18.677074 29126 fs_report.cc:389] FS layout report
02:04:18.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:04:18.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/ts-0/wal
02:04:18.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/ts-0/wal
02:04:18.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/ts-0/data/data
02:04:18.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:04:18.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:04:18.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:04:18.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:04:18.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:04:18.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:04:18.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:04:18.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:04:18.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:04:18.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:04:18.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:04:18.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:18.796751 29126 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:04:18.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:18.814071 29126 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:04:18.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:18.814888 29126 kserver.cc:163] Server-wide thread pool size limit: 3276
02:04:18.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:18.828836 29126 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
02:04:18.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:18.833943 29126 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
02:04:18.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:18.834197 29126 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
02:04:18.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:18.834488 29126 ts_tablet_manager.cc:616] Registered 0 tablets
02:04:18.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:18.834700 29126 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
02:04:19.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 17 02:04:19 dist-test-slave-32ct krb5kdc[28797](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781661859, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.28.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:04:19.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:19.053444 28829 init.cc:295] Successfully reacquired a new kerberos TGT
02:04:19.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:19.104282 29126 rpc_server.cc:307] RPC server started. Bound to: 127.28.28.1:40193
02:04:19.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:19.105106 29261 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.28.28.1:40193 every 8 connection(s)
02:04:19.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:19.108397 29126 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/ts-0/data/info.pb
02:04:19.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:19.111869 28784 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskqQ5_Rr/build/tsan/bin/kudu as pid 29126
02:04:19.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:19.113566 28784 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/ts-0/wal/instance
02:04:19.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 17 02:04:19 dist-test-slave-32ct krb5kdc[28797](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781661858, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.28.1@KRBTEST.COM for kudu/127.28.28.60@KRBTEST.COM
02:04:19.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:19.216615 29262 heartbeater.cc:344] Connected to a master server at 127.28.28.60:37309
02:04:19.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:19.217109 29262 heartbeater.cc:461] Registering TS with master...
02:04:19.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:19.218441 29262 heartbeater.cc:507] Master 127.28.28.60:37309 requested a full tablet report, sending...
02:04:19.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 17 02:04:19 dist-test-slave-32ct krb5kdc[28797](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781661858, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.28.1@KRBTEST.COM for kudu/127.28.28.61@KRBTEST.COM
02:04:19.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:19.221962 29023 ts_manager.cc:194] Registered new tserver with Master: 232880c2803f4de0851f2c9578cf2128 (127.28.28.1:40193)
02:04:19.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 17 02:04:19 dist-test-slave-32ct krb5kdc[28797](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781661858, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.28.1@KRBTEST.COM for kudu/127.28.28.62@KRBTEST.COM
02:04:19.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.28.28.2@KRBTEST.COM; defaulting to no policy
02:04:19.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:04:19.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.28.28.2@KRBTEST.COM" created.
02:04:19.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:19.237973 29265 heartbeater.cc:344] Connected to a master server at 127.28.28.61:38085
02:04:19.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:19.238306 29265 heartbeater.cc:461] Registering TS with master...
02:04:19.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:19.239205 29265 heartbeater.cc:507] Master 127.28.28.61:38085 requested a full tablet report, sending...
02:04:19.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:19.242338 28936 ts_manager.cc:194] Registered new tserver with Master: 232880c2803f4de0851f2c9578cf2128 (127.28.28.1:40193)
02:04:19.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:19.252244 29266 heartbeater.cc:344] Connected to a master server at 127.28.28.62:46845
02:04:19.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:19.252596 29266 heartbeater.cc:461] Registering TS with master...
02:04:19.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:19.253674 29266 heartbeater.cc:507] Master 127.28.28.62:46845 requested a full tablet report, sending...
02:04:19.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:19.258316 28851 ts_manager.cc:194] Registered new tserver with Master: 232880c2803f4de0851f2c9578cf2128 (127.28.28.1:40193)
02:04:19.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:19.262529 28851 master_service.cc:502] Signed X509 certificate for tserver {username='kudu', principal='kudu/127.28.28.1@KRBTEST.COM'} at 127.28.28.1:38207
02:04:19.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:04:19.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.28.28.2 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/krb5kdc/kudu.keytab.
02:04:19.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.28.28.2 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/krb5kdc/kudu.keytab.
02:04:19.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:19.281342 28784 mini_kdc.cc:268] Time spent creating service keytab for kudu/127.28.28.2: real 0.145s user 0.000s sys 0.014s
02:04:19.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.28.28.2@KRBTEST.COM; defaulting to no policy
02:04:19.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:04:19.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.28.28.2@KRBTEST.COM" created.
02:04:19.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:04:19.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.28.28.2 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/krb5kdc/kudu.keytab.
02:04:19.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.28.28.2 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/krb5kdc/kudu.keytab.
02:04:19.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:19.344691 28784 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskqQ5_Rr/build/tsan/bin/kudu
02:04:19.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskqQ5_Rr/build/tsan/bin/kudu
02:04:19.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/ts-1/wal
02:04:19.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/ts-1/data
02:04:19.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:04:19.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:04:19.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:04:19.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:04:19.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:04:19.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:04:19.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/ts-1/logs
02:04:19.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/ts-1/data/info.pb
02:04:19.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:04:19.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:04:19.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:04:19.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:04:19.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:04:19.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:04:19.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:04:19.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:04:19.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
02:04:19.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:04:19.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.28.2:0
02:04:19.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.28.28.2
02:04:19.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.28.2
02:04:19.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:04:19.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.28.28.62:46845,127.28.28.61:38085,127.28.28.60:37309
02:04:19.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/krb5kdc/kudu.keytab
02:04:19.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.28.2
02:04:19.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:04:19.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
02:04:19.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
02:04:19.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
02:04:19.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.28.20:40741
02:04:19.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:04:19.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:04:19.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:04:19.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false with env {KRB5CCNAME=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
02:04:19.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:19.784008 29276 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
02:04:19.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:19.785125 29276 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:04:19.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:19.785923 29276 flags.cc:432] Enabled unsafe flag: --never_fsync=true
02:04:19.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:19.820216 29276 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
02:04:19.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:19.821312 29276 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.28.28.2
02:04:19.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:19.863101 29276 tablet_server_runner.cc:78] Tablet server non-default flags:
02:04:19.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:04:19.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.28.20:40741
02:04:19.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:04:19.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:04:19.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/ts-1/data
02:04:19.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/ts-1/wal
02:04:19.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/krb5kdc/kudu.keytab
02:04:19.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.28.2
02:04:19.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:04:19.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:04:19.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.28.2:0
02:04:19.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:04:19.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:04:19.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
02:04:19.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
02:04:19.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:04:19.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:04:19.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/ts-1/data/info.pb
02:04:19.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.28.2
02:04:19.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:04:19.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.28.28.62:46845,127.28.28.61:38085,127.28.28.60:37309
02:04:19.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:04:19.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:04:19.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:04:19.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:04:19.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:04:19.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.28.28.2
02:04:19.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/ts-1/logs
02:04:19.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:04:19.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:04:19.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:04:19.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
02:04:19.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:04:19.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 23b10557f461bc2be3d2c282748d8de96f9c46ec
02:04:19.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:04:19.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 17 Jun 2026 01:43:23 UTC on e7f111948823
02:04:19.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12431
02:04:19.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:04:19.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:19.864825 29276 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:04:19.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:19.866245 29276 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:04:19.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:19.880631 29284 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:04:20.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:20.266940 29266 heartbeater.cc:499] Master 127.28.28.62:46845 was elected leader, sending a full tablet report...
02:04:21.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:19.891659 29287 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:04:21.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:19.888965 29285 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:04:21.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:21.134958 29286 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
02:04:21.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:21.135018 29276 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:04:21.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:21.139250 29276 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:04:21.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:21.141690 29276 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:04:21.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:21.143028 29276 hybrid_clock.cc:648] HybridClock initialized: now 1781661861142991 us; error 43 us; skew 500 ppm
02:04:21.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:21.143975 29276 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:04:21.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 17 02:04:21 dist-test-slave-32ct krb5kdc[28797](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781661861, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.28.2@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:04:21.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:21.151692 29276 init.cc:377] Logged in from keytab as kudu/127.28.28.2@KRBTEST.COM (short username kudu)
02:04:21.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:21.158049 29276 webserver.cc:533] Webserver started at http://127.28.28.2:46489/ using document root <none> and password file <none>
02:04:21.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:21.159040 29276 fs_manager.cc:362] Metadata directory not provided
02:04:21.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:21.159291 29276 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:04:21.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:21.159845 29276 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
02:04:21.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:21.164917 29276 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/ts-1/data/instance:
02:04:21.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6ad9f08d43c343e1b54840b7c9e52ac9"
02:04:21.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-17 02:04:21 on dist-test-slave-32ct"
02:04:21.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:21.166146 29276 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/ts-1/wal/instance:
02:04:21.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6ad9f08d43c343e1b54840b7c9e52ac9"
02:04:21.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-17 02:04:21 on dist-test-slave-32ct"
02:04:21.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:21.175441 29276 fs_manager.cc:696] Time spent creating directory manager: real 0.009s user 0.005s sys 0.004s
02:04:21.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:21.182262 29298 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:04:21.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:21.183358 29276 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.004s sys 0.000s
02:04:21.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:21.183725 29276 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/ts-1/data,/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/ts-1/wal
02:04:21.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6ad9f08d43c343e1b54840b7c9e52ac9"
02:04:21.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-17 02:04:21 on dist-test-slave-32ct"
02:04:21.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:21.184069 29276 fs_report.cc:389] FS layout report
02:04:21.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:04:21.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/ts-1/wal
02:04:21.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/ts-1/wal
02:04:21.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/ts-1/data/data
02:04:21.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:04:21.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:04:21.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:04:21.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:04:21.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:04:21.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:04:21.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:04:21.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:04:21.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:04:21.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:04:21.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:04:21.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:21.254631 29276 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:04:21.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:21.262423 29276 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:04:21.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:21.262934 29276 kserver.cc:163] Server-wide thread pool size limit: 3276
02:04:21.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:21.266165 29276 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
02:04:21.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:21.272040 29276 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
02:04:21.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:21.272316 29276 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
02:04:21.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:21.272610 29276 ts_tablet_manager.cc:616] Registered 0 tablets
02:04:21.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:21.272845 29276 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.001s sys 0.000s
02:04:21.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:21.452479 29276 rpc_server.cc:307] RPC server started. Bound to: 127.28.28.2:33169
02:04:21.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:21.452589 29410 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.28.28.2:33169 every 8 connection(s)
02:04:21.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:21.455655 29276 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/ts-1/data/info.pb
02:04:21.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:21.465040 28784 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskqQ5_Rr/build/tsan/bin/kudu as pid 29276
02:04:21.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:21.465677 28784 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/ts-1/wal/instance
02:04:21.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.28.28.3@KRBTEST.COM; defaulting to no policy
02:04:21.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:04:21.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.28.28.3@KRBTEST.COM" created.
02:04:21.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 17 02:04:21 dist-test-slave-32ct krb5kdc[28797](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781661861, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.28.2@KRBTEST.COM for kudu/127.28.28.62@KRBTEST.COM
02:04:21.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 17 02:04:21 dist-test-slave-32ct krb5kdc[28797](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781661861, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.28.2@KRBTEST.COM for kudu/127.28.28.60@KRBTEST.COM
02:04:21.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 17 02:04:21 dist-test-slave-32ct krb5kdc[28797](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781661861, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.28.2@KRBTEST.COM for kudu/127.28.28.61@KRBTEST.COM
02:04:21.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 17 02:04:21 dist-test-slave-32ct krb5kdc[28797](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781661861, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.28.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:04:21.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:21.551566 28913 init.cc:295] Successfully reacquired a new kerberos TGT
02:04:21.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:21.562170 29411 heartbeater.cc:344] Connected to a master server at 127.28.28.60:37309
02:04:21.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:21.562676 29411 heartbeater.cc:461] Registering TS with master...
02:04:21.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:21.563879 29411 heartbeater.cc:507] Master 127.28.28.60:37309 requested a full tablet report, sending...
02:04:21.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:04:21.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.28.28.3 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/krb5kdc/kudu.keytab.
02:04:21.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.28.28.3 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/krb5kdc/kudu.keytab.
02:04:21.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:21.567232 29023 ts_manager.cc:194] Registered new tserver with Master: 6ad9f08d43c343e1b54840b7c9e52ac9 (127.28.28.2:33169)
02:04:21.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:21.571420 29412 heartbeater.cc:344] Connected to a master server at 127.28.28.61:38085
02:04:21.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:21.571790 29412 heartbeater.cc:461] Registering TS with master...
02:04:21.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:21.573032 29413 heartbeater.cc:344] Connected to a master server at 127.28.28.62:46845
02:04:21.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:21.573529 29413 heartbeater.cc:461] Registering TS with master...
02:04:21.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:21.573818 29412 heartbeater.cc:507] Master 127.28.28.61:38085 requested a full tablet report, sending...
02:04:21.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:21.575603 29413 heartbeater.cc:507] Master 127.28.28.62:46845 requested a full tablet report, sending...
02:04:21.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:21.577672 28936 ts_manager.cc:194] Registered new tserver with Master: 6ad9f08d43c343e1b54840b7c9e52ac9 (127.28.28.2:33169)
02:04:21.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:21.579849 28851 ts_manager.cc:194] Registered new tserver with Master: 6ad9f08d43c343e1b54840b7c9e52ac9 (127.28.28.2:33169)
02:04:21.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:21.581530 28851 master_service.cc:502] Signed X509 certificate for tserver {username='kudu', principal='kudu/127.28.28.2@KRBTEST.COM'} at 127.28.28.2:44413
02:04:21.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.28.28.3@KRBTEST.COM; defaulting to no policy
02:04:21.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:04:21.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.28.28.3@KRBTEST.COM" created.
02:04:21.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:04:21.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.28.28.3 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/krb5kdc/kudu.keytab.
02:04:21.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.28.28.3 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/krb5kdc/kudu.keytab.
02:04:21.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:21.607560 28784 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskqQ5_Rr/build/tsan/bin/kudu
02:04:21.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskqQ5_Rr/build/tsan/bin/kudu
02:04:21.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/ts-2/wal
02:04:21.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/ts-2/data
02:04:21.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:04:21.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:04:21.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:04:21.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:04:21.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:04:21.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:04:21.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/ts-2/logs
02:04:21.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/ts-2/data/info.pb
02:04:21.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:04:21.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:04:21.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:04:21.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:04:21.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:04:21.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:04:21.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:04:21.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:04:21.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
02:04:21.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:04:21.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.28.3:0
02:04:21.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.28.28.3
02:04:21.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.28.3
02:04:21.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:04:21.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.28.28.62:46845,127.28.28.61:38085,127.28.28.60:37309
02:04:21.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/krb5kdc/kudu.keytab
02:04:21.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.28.3
02:04:21.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:04:21.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
02:04:21.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
02:04:21.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
02:04:21.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.28.20:40741
02:04:21.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:04:21.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:04:21.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:04:21.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false with env {KRB5CCNAME=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
02:04:22.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:22.211833 29425 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
02:04:22.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:22.212464 29425 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:04:22.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:22.213090 29425 flags.cc:432] Enabled unsafe flag: --never_fsync=true
02:04:22.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:22.248159 29425 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
02:04:22.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:22.249096 29425 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.28.28.3
02:04:22.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:22.298277 29425 tablet_server_runner.cc:78] Tablet server non-default flags:
02:04:22.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:04:22.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.28.20:40741
02:04:22.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:04:22.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:04:22.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/ts-2/data
02:04:22.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/ts-2/wal
02:04:22.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/krb5kdc/kudu.keytab
02:04:22.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.28.3
02:04:22.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:04:22.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:04:22.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.28.3:0
02:04:22.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:04:22.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:04:22.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
02:04:22.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
02:04:22.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:04:22.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:04:22.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/ts-2/data/info.pb
02:04:22.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.28.3
02:04:22.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:04:22.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.28.28.62:46845,127.28.28.61:38085,127.28.28.60:37309
02:04:22.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:04:22.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:04:22.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:04:22.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:04:22.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:04:22.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.28.28.3
02:04:22.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/ts-2/logs
02:04:22.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:04:22.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:04:22.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:04:22.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
02:04:22.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:04:22.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 23b10557f461bc2be3d2c282748d8de96f9c46ec
02:04:22.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:04:22.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 17 Jun 2026 01:43:23 UTC on e7f111948823
02:04:22.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12431
02:04:22.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:04:22.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:22.299531 29425 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:04:22.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:22.300766 29425 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:04:22.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:22.319236 29433 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:04:22.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:22.584676 29413 heartbeater.cc:499] Master 127.28.28.62:46845 was elected leader, sending a full tablet report...
02:04:22.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:22.971302 28628 debug-util.cc:398] Leaking SignalData structure 0x7b08000ade40 after lost signal to thread 28503
02:04:22.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:22.973071 28628 debug-util.cc:398] Leaking SignalData structure 0x7b08000bab80 after lost signal to thread 28631
02:04:23.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:22.323153 29434 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:04:23.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:23.615906 29435 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
02:04:23.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:23.615933 29425 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:04:23.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:22.329967 29437 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:04:23.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:23.621534 29425 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:04:23.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:23.624558 29425 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:04:23.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:23.625973 29425 hybrid_clock.cc:648] HybridClock initialized: now 1781661863625912 us; error 78 us; skew 500 ppm
02:04:23.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:23.626727 29425 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:04:23.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 17 02:04:23 dist-test-slave-32ct krb5kdc[28797](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781661863, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.28.3@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:04:23.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:23.633160 29425 init.cc:377] Logged in from keytab as kudu/127.28.28.3@KRBTEST.COM (short username kudu)
02:04:23.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:23.639125 29425 webserver.cc:533] Webserver started at http://127.28.28.3:46439/ using document root <none> and password file <none>
02:04:23.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:23.640175 29425 fs_manager.cc:362] Metadata directory not provided
02:04:23.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:23.640445 29425 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:04:23.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:23.640956 29425 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
02:04:23.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:23.645314 29425 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/ts-2/data/instance:
02:04:23.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2c49f65da14341358a43e3ad6e67541a"
02:04:23.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-17 02:04:23 on dist-test-slave-32ct"
02:04:23.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:23.646654 29425 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/ts-2/wal/instance:
02:04:23.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2c49f65da14341358a43e3ad6e67541a"
02:04:23.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-17 02:04:23 on dist-test-slave-32ct"
02:04:23.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:23.654582 29425 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.008s sys 0.000s
02:04:23.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:23.660766 29445 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:04:23.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:23.661901 29425 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.001s
02:04:23.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:23.662174 29425 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/ts-2/data,/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/ts-2/wal
02:04:23.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2c49f65da14341358a43e3ad6e67541a"
02:04:23.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-17 02:04:23 on dist-test-slave-32ct"
02:04:23.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:23.662454 29425 fs_report.cc:389] FS layout report
02:04:23.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:04:23.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/ts-2/wal
02:04:23.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/ts-2/wal
02:04:23.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/ts-2/data/data
02:04:23.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:04:23.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:04:23.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:04:23.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:04:23.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:04:23.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:04:23.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:04:23.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:04:23.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:04:23.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:04:23.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:04:23.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:23.722620 29425 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:04:23.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:23.730018 29425 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:04:23.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:23.730568 29425 kserver.cc:163] Server-wide thread pool size limit: 3276
02:04:23.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:23.744558 29425 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
02:04:23.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:23.756462 29425 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
02:04:23.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:23.756736 29425 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
02:04:23.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:23.757035 29425 ts_tablet_manager.cc:616] Registered 0 tablets
02:04:23.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:23.757257 29425 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
02:04:23.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 17 02:04:23 dist-test-slave-32ct krb5kdc[28797](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781661863, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.28.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:04:23.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:23.880496 29002 init.cc:295] Successfully reacquired a new kerberos TGT
02:04:24.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:24.028887 29425 rpc_server.cc:307] RPC server started. Bound to: 127.28.28.3:45433
02:04:24.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:24.029035 29558 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.28.28.3:45433 every 8 connection(s)
02:04:24.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:24.032557 29425 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/ts-2/data/info.pb
02:04:24.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:24.036808 28784 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskqQ5_Rr/build/tsan/bin/kudu as pid 29425
02:04:24.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:24.037295 28784 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/ts-2/wal/instance
02:04:24.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 17 02:04:24 dist-test-slave-32ct krb5kdc[28797](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781661863, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.28.3@KRBTEST.COM for kudu/127.28.28.61@KRBTEST.COM
02:04:24.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 17 02:04:24 dist-test-slave-32ct krb5kdc[28797](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781661863, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.28.3@KRBTEST.COM for kudu/127.28.28.60@KRBTEST.COM
02:04:24.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 17 02:04:24 dist-test-slave-32ct krb5kdc[28797](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781661863, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.28.3@KRBTEST.COM for kudu/127.28.28.62@KRBTEST.COM
02:04:24.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:24.176220 29560 heartbeater.cc:344] Connected to a master server at 127.28.28.61:38085
02:04:24.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:24.176771 29560 heartbeater.cc:461] Registering TS with master...
02:04:24.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:24.178215 29560 heartbeater.cc:507] Master 127.28.28.61:38085 requested a full tablet report, sending...
02:04:24.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:24.179924 29559 heartbeater.cc:344] Connected to a master server at 127.28.28.60:37309
02:04:24.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:24.180282 29559 heartbeater.cc:461] Registering TS with master...
02:04:24.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:24.181227 29559 heartbeater.cc:507] Master 127.28.28.60:37309 requested a full tablet report, sending...
02:04:24.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:24.182080 28936 ts_manager.cc:194] Registered new tserver with Master: 2c49f65da14341358a43e3ad6e67541a (127.28.28.3:45433)
02:04:24.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:24.182132 29561 heartbeater.cc:344] Connected to a master server at 127.28.28.62:46845
02:04:24.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:24.182511 29561 heartbeater.cc:461] Registering TS with master...
02:04:24.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:24.183447 29561 heartbeater.cc:507] Master 127.28.28.62:46845 requested a full tablet report, sending...
02:04:24.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:24.184307 29023 ts_manager.cc:194] Registered new tserver with Master: 2c49f65da14341358a43e3ad6e67541a (127.28.28.3:45433)
02:04:24.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:24.185634 28851 ts_manager.cc:194] Registered new tserver with Master: 2c49f65da14341358a43e3ad6e67541a (127.28.28.3:45433)
02:04:24.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:24.187409 28851 master_service.cc:502] Signed X509 certificate for tserver {username='kudu', principal='kudu/127.28.28.3@KRBTEST.COM'} at 127.28.28.3:50899
02:04:24.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:24.198303 28784 external_mini_cluster.cc:955] 3 TS(s) registered with all masters
02:04:24.214 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:04:24.215 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
02:04:24.224 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
02:04:24.225 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
02:04:24.229 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.28.28.62"
port: 46845
}
bound_http_address {
host: "127.28.28.62"
port: 35423
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.28.28.61"
port: 38085
}
bound_http_address {
host: "127.28.28.61"
port: 34599
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.28.28.60"
port: 37309
}
bound_http_address {
host: "127.28.28.60"
port: 38177
}
}
}
02:04:24.229 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
02:04:24.230 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.28.28.1"
port: 40193
}
bound_http_address {
host: "127.28.28.1"
port: 45263
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.28.28.2"
port: 33169
}
bound_http_address {
host: "127.28.28.2"
port: 46489
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.28.28.3"
port: 45433
}
bound_http_address {
host: "127.28.28.3"
port: 46439
}
}
}
02:04:24.231 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
02:04:24.232 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
02:04:24.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 17 02:04:24 dist-test-slave-32ct krb5kdc[28797](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781661864, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:04:24.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
02:04:24.274 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:04:24.277 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/krb5kdc/krb5cc
02:04:24.290 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
02:04:24.290 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/krb5kdc/krb5cc
02:04:24.292 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
02:04:24.328 [DEBUG - Time-limited test] (SecurityContext.java:158) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
02:04:24.330 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
02:04:24.331 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
02:04:24.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 17 02:04:24 dist-test-slave-32ct krb5kdc[28797](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781661864, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:04:24.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
02:04:24.372 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:04:24.372 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/krb5kdc/krb5cc
02:04:24.374 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
02:04:24.375 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.28.28.62:46845
02:04:24.376 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
02:04:24.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:24.377316 28784 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskqQ5_Rr/build/tsan/bin/kudu with pid 28816
02:04:24.420 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:04:24.420 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.28.28.61:38085
02:04:24.421 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
02:04:24.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:24.421298 28784 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskqQ5_Rr/build/tsan/bin/kudu with pid 28895
02:04:24.458 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:04:24.458 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.28.28.60:37309
02:04:24.459 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
02:04:24.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:24.459503 28784 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskqQ5_Rr/build/tsan/bin/kudu with pid 28981
02:04:24.496 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:04:24.496 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.28.28.62:46845
02:04:24.500 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
02:04:24.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:24.502403 28784 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskqQ5_Rr/build/tsan/bin/kudu
02:04:24.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskqQ5_Rr/build/tsan/bin/kudu
02:04:24.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/master-0/wal
02:04:24.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/master-0/data
02:04:24.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:04:24.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:04:24.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:04:24.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:04:24.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:04:24.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:04:24.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/master-0/logs
02:04:24.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/master-0/data/info.pb
02:04:24.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:04:24.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:04:24.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:04:24.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:04:24.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:04:24.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:04:24.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:04:24.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:04:24.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:04:24.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:04:24.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:04:24.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:04:24.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.28.62:46845
02:04:24.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.28.62
02:04:24.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35423
02:04:24.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/krb5kdc/kudu.keytab
02:04:24.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.28.62
02:04:24.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:04:24.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
02:04:24.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
02:04:24.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
02:04:24.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.28.20:40741
02:04:24.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:04:24.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:04:24.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:04:24.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:04:24.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.28.62:46845,127.28.28.61:38085,127.28.28.60:37309
02:04:24.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:04:24.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35823/jwks.json
02:04:24.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
02:04:24.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
02:04:24.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:24.581647 29411 heartbeater.cc:646] Failed to heartbeat to 127.28.28.60:37309 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.28.28.60:37309: connect: Connection refused (error 111)
02:04:24.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:24.851639 29573 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
02:04:24.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:24.852326 29573 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:04:24.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:24.853176 29573 flags.cc:432] Enabled unsafe flag: --never_fsync=true
02:04:24.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:24.889812 29573 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
02:04:24.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:24.890098 29573 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
02:04:24.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:24.890527 29573 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
02:04:24.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:24.890868 29573 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
02:04:24.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:24.891199 29573 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
02:04:24.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:24.891518 29573 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
02:04:24.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:24.891808 29573 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:35823/jwks.json
02:04:24.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:24.932602 29573 master_runner.cc:387] Master server non-default flags:
02:04:24.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:04:24.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.28.20:40741
02:04:24.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:04:24.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:04:24.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/master-0/data
02:04:24.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/master-0/wal
02:04:24.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
02:04:24.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:04:24.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.28.62:46845,127.28.28.61:38085,127.28.28.60:37309
02:04:24.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/krb5kdc/kudu.keytab
02:04:24.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.28.62
02:04:24.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:04:24.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:04:24.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:04:24.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.28.62:46845
02:04:24.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:04:24.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:04:24.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:04:24.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35823/jwks.json
02:04:24.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:04:24.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
02:04:24.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
02:04:24.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:04:24.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:04:24.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/master-0/data/info.pb
02:04:24.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.28.62
02:04:24.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35423
02:04:24.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:04:24.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:04:24.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:04:24.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:04:24.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:04:24.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/master-0/logs
02:04:24.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:04:24.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:04:24.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:04:24.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:04:24.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:04:24.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 23b10557f461bc2be3d2c282748d8de96f9c46ec
02:04:24.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:04:24.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 17 Jun 2026 01:43:23 UTC on e7f111948823
02:04:24.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12431
02:04:24.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:04:24.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:24.934259 29573 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:04:24.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:24.936326 29573 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:04:24.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:24.949235 29581 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:04:25.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:25.197748 29561 heartbeater.cc:499] Master 127.28.28.62:46845 was elected leader, sending a full tablet report...
02:04:25.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:25.201586 29559 heartbeater.cc:646] Failed to heartbeat to 127.28.28.60:37309 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.28.28.60:37309: connect: Connection refused (error 111)
02:04:25.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:25.272893 29262 heartbeater.cc:646] Failed to heartbeat to 127.28.28.60:37309 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.28.28.60:37309: connect: Connection refused (error 111)
02:04:26.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:26.223726 28291 debug-util.cc:398] Leaking SignalData structure 0x7b0800081200 after lost signal to thread 28221
02:04:26.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:26.224788 28291 debug-util.cc:398] Leaking SignalData structure 0x7b080009e020 after lost signal to thread 28294
02:04:26.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:24.951988 29582 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:04:26.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:26.356503 29580 debug-util.cc:398] Leaking SignalData structure 0x7b080002ef00 after lost signal to thread 29573
02:04:26.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:26.855545 29583 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1903 milliseconds
02:04:26.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:26.854758 29573 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.905s user 0.553s sys 0.931s
02:04:26.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:26.856704 29573 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.907s user 0.553s sys 0.931s
02:04:26.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:26.858048 29584 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:04:26.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:26.858103 29573 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:04:26.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:26.860847 29573 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:04:26.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:26.863379 29573 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:04:26.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:26.864738 29573 hybrid_clock.cc:648] HybridClock initialized: now 1781661866864704 us; error 54 us; skew 500 ppm
02:04:26.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:26.865655 29573 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:04:26.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 17 02:04:26 dist-test-slave-32ct krb5kdc[28797](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781661866, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.28.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:04:26.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:26.872007 29573 init.cc:377] Logged in from keytab as kudu/127.28.28.62@KRBTEST.COM (short username kudu)
02:04:26.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:26.877612 29573 webserver.cc:533] Webserver started at http://127.28.28.62:35423/ using document root <none> and password file <none>
02:04:26.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:26.878540 29573 fs_manager.cc:362] Metadata directory not provided
02:04:26.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:26.878793 29573 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:04:26.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:26.887192 29573 fs_manager.cc:714] Time spent opening directory manager: real 0.005s user 0.002s sys 0.002s
02:04:26.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:26.892110 29604 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:04:26.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:26.893275 29573 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.004s sys 0.000s
02:04:26.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:26.894695 28768 debug-util.cc:398] Leaking SignalData structure 0x7b08000ad1e0 after lost signal to thread 28643
02:04:26.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:26.895900 28768 debug-util.cc:398] Leaking SignalData structure 0x7b08000b99a0 after lost signal to thread 28771
02:04:26.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:26.893690 29573 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/master-0/data,/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/master-0/wal
02:04:26.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4ce4fbc4c4344da38c81053f87cf6862"
02:04:26.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-17 02:04:09 on dist-test-slave-32ct"
02:04:26.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:26.895788 29573 fs_report.cc:389] FS layout report
02:04:26.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:04:26.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/master-0/wal
02:04:26.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/master-0/wal
02:04:26.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/master-0/data/data
02:04:26.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:04:26.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:04:26.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:04:26.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:04:26.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:04:26.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:04:26.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:04:26.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:04:26.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:04:26.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:04:26.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:04:26.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:26.963604 29573 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:04:26.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:26.969645 29573 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:04:26.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:26.970197 29573 kserver.cc:163] Server-wide thread pool size limit: 3276
02:04:27.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:27.147361 29573 rpc_server.cc:307] RPC server started. Bound to: 127.28.28.62:46845
02:04:27.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:27.147951 29655 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.28.28.62:46845 every 8 connection(s)
02:04:27.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:27.150964 29573 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/master-0/data/info.pb
02:04:27.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:27.160862 28784 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskqQ5_Rr/build/tsan/bin/kudu as pid 29573
02:04:27.162 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:04:27.162 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.28.28.61:38085
02:04:27.162 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
02:04:27.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:27.164979 28784 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskqQ5_Rr/build/tsan/bin/kudu
02:04:27.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskqQ5_Rr/build/tsan/bin/kudu
02:04:27.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/master-1/wal
02:04:27.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/master-1/data
02:04:27.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:04:27.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:04:27.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:04:27.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:04:27.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:04:27.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:04:27.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/master-1/logs
02:04:27.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/master-1/data/info.pb
02:04:27.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:04:27.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:04:27.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:04:27.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:04:27.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:04:27.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:04:27.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:04:27.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:04:27.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:04:27.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:04:27.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:04:27.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:04:27.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.28.61:38085
02:04:27.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.28.61
02:04:27.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=34599
02:04:27.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/krb5kdc/kudu.keytab
02:04:27.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.28.61
02:04:27.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:04:27.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
02:04:27.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
02:04:27.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
02:04:27.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.28.20:40741
02:04:27.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:04:27.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:04:27.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:04:27.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:04:27.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.28.62:46845,127.28.28.61:38085,127.28.28.60:37309
02:04:27.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:04:27.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35823/jwks.json
02:04:27.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
02:04:27.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
02:04:27.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:27.176625 29656 sys_catalog.cc:263] Verifying existing consensus state
02:04:27.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:27.185472 29656 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4ce4fbc4c4344da38c81053f87cf6862: Bootstrap starting.
02:04:27.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:27.249554 29656 log.cc:826] T 00000000000000000000000000000000 P 4ce4fbc4c4344da38c81053f87cf6862: Log is configured to *not* fsync() on all Append() calls
02:04:27.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:27.338784 29656 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4ce4fbc4c4344da38c81053f87cf6862: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
02:04:27.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:27.339869 29656 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4ce4fbc4c4344da38c81053f87cf6862: Bootstrap complete.
02:04:27.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:27.376302 29561 heartbeater.cc:344] Connected to a master server at 127.28.28.62:46845
02:04:27.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:27.376801 29561 heartbeater.cc:499] Master 127.28.28.62:46845 was elected leader, sending a full tablet report...
02:04:27.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:27.410334 29266 heartbeater.cc:344] Connected to a master server at 127.28.28.62:46845
02:04:27.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:27.404346 29656 raft_consensus.cc:359] T 00000000000000000000000000000000 P 4ce4fbc4c4344da38c81053f87cf6862 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4ce4fbc4c4344da38c81053f87cf6862" member_type: VOTER last_known_addr { host: "127.28.28.62" port: 46845 } } peers { permanent_uuid: "2c6224a4528c494eb782e1c4e5a71288" member_type: VOTER last_known_addr { host: "127.28.28.61" port: 38085 } } peers { permanent_uuid: "698282ccaf55491badb00732fef7a918" member_type: VOTER last_known_addr { host: "127.28.28.60" port: 37309 } }
02:04:27.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:27.414034 29656 raft_consensus.cc:740] T 00000000000000000000000000000000 P 4ce4fbc4c4344da38c81053f87cf6862 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4ce4fbc4c4344da38c81053f87cf6862, State: Initialized, Role: FOLLOWER
02:04:27.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:27.415246 29656 consensus_queue.cc:260] T 00000000000000000000000000000000 P 4ce4fbc4c4344da38c81053f87cf6862 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4ce4fbc4c4344da38c81053f87cf6862" member_type: VOTER last_known_addr { host: "127.28.28.62" port: 46845 } } peers { permanent_uuid: "2c6224a4528c494eb782e1c4e5a71288" member_type: VOTER last_known_addr { host: "127.28.28.61" port: 38085 } } peers { permanent_uuid: "698282ccaf55491badb00732fef7a918" member_type: VOTER last_known_addr { host: "127.28.28.60" port: 37309 } }
02:04:27.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:27.432973 29666 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4ce4fbc4c4344da38c81053f87cf6862 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4ce4fbc4c4344da38c81053f87cf6862" member_type: VOTER last_known_addr { host: "127.28.28.62" port: 46845 } } peers { permanent_uuid: "2c6224a4528c494eb782e1c4e5a71288" member_type: VOTER last_known_addr { host: "127.28.28.61" port: 38085 } } peers { permanent_uuid: "698282ccaf55491badb00732fef7a918" member_type: VOTER last_known_addr { host: "127.28.28.60" port: 37309 } } }
02:04:27.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:27.433956 29666 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4ce4fbc4c4344da38c81053f87cf6862 [sys.catalog]: This master's current role is: FOLLOWER
02:04:27.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:27.436275 29656 sys_catalog.cc:565] T 00000000000000000000000000000000 P 4ce4fbc4c4344da38c81053f87cf6862 [sys.catalog]: configured and running, proceeding with master startup.
02:04:27.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:27.528855 29680 catalog_manager.cc:805] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
02:04:27.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 17 02:04:27 dist-test-slave-32ct krb5kdc[28797](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781661866, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.28.62@KRBTEST.COM for kudu/127.28.28.62@KRBTEST.COM
02:04:27.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:27.603031 29573 master_runner.cc:187] Error getting master registration for 127.28.28.61:38085: Network error: Client connection negotiation failed: client connection to 127.28.28.61:38085: connect: Connection refused (error 111),
02:04:27.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:27.666764 29573 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
02:04:27.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:27.681969 29413 heartbeater.cc:344] Connected to a master server at 127.28.28.62:46845
02:04:27.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:27.688176 29621 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "6ad9f08d43c343e1b54840b7c9e52ac9" instance_seqno: 1781661861418128) as {username='kudu', principal='kudu/127.28.28.2@KRBTEST.COM'} at 127.28.28.2:50335; Asking this server to re-register.
02:04:27.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:27.690102 29413 heartbeater.cc:461] Registering TS with master...
02:04:27.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:27.693558 29620 ts_manager.cc:194] Registered new tserver with Master: 6ad9f08d43c343e1b54840b7c9e52ac9 (127.28.28.2:33169)
02:04:27.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:27.841056 29658 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
02:04:27.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:27.841917 29658 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:04:27.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:27.843086 29658 flags.cc:432] Enabled unsafe flag: --never_fsync=true
02:04:27.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:27.913928 29658 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
02:04:27.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:27.914278 29658 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
02:04:27.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:27.914798 29658 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
02:04:27.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:27.915170 29658 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
02:04:27.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:27.915537 29658 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
02:04:27.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:27.915904 29658 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
02:04:27.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:27.916181 29658 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:35823/jwks.json
02:04:27.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:27.998203 29658 master_runner.cc:387] Master server non-default flags:
02:04:27.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:04:27.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.28.20:40741
02:04:28.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:04:28.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:04:28.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/master-1/data
02:04:28.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/master-1/wal
02:04:28.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
02:04:28.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:04:28.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.28.62:46845,127.28.28.61:38085,127.28.28.60:37309
02:04:28.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/krb5kdc/kudu.keytab
02:04:28.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.28.61
02:04:28.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:04:28.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:04:28.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:04:28.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.28.61:38085
02:04:28.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:04:28.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:04:28.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:04:28.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35823/jwks.json
02:04:28.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:04:28.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
02:04:28.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
02:04:28.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:04:28.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:04:28.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/master-1/data/info.pb
02:04:28.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.28.61
02:04:28.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=34599
02:04:28.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:04:28.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:04:28.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:04:28.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:04:28.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:04:28.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/master-1/logs
02:04:28.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:04:28.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:04:28.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:04:28.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:04:28.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:04:28.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 23b10557f461bc2be3d2c282748d8de96f9c46ec
02:04:28.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:04:28.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 17 Jun 2026 01:43:23 UTC on e7f111948823
02:04:28.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12431
02:04:28.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:04:28.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:28.000751 29658 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:04:28.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:28.002173 29658 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:04:28.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:28.017875 29690 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:04:28.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:28.381389 29561 heartbeater.cc:499] Master 127.28.28.62:46845 was elected leader, sending a full tablet report...
02:04:28.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:28.384972 29620 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "2c49f65da14341358a43e3ad6e67541a" instance_seqno: 1781661863980528) as {username='kudu', principal='kudu/127.28.28.3@KRBTEST.COM'} at 127.28.28.3:44555; Asking this server to re-register.
02:04:28.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:28.386899 29561 heartbeater.cc:461] Registering TS with master...
02:04:28.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:28.390545 29620 ts_manager.cc:194] Registered new tserver with Master: 2c49f65da14341358a43e3ad6e67541a (127.28.28.3:45433)
02:04:28.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:28.416604 29620 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "232880c2803f4de0851f2c9578cf2128" instance_seqno: 1781661859014261) as {username='kudu', principal='kudu/127.28.28.1@KRBTEST.COM'} at 127.28.28.1:53323; Asking this server to re-register.
02:04:28.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:28.418655 29266 heartbeater.cc:461] Registering TS with master...
02:04:28.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:28.423580 29620 ts_manager.cc:194] Registered new tserver with Master: 232880c2803f4de0851f2c9578cf2128 (127.28.28.1:40193)
02:04:28.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:28.557606 29680 catalog_manager.cc:1294] Loaded cluster ID: f94c5ca418a8467e9c05f9f73c940f49
02:04:28.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:28.557893 29680 catalog_manager.cc:1587] T 00000000000000000000000000000000 P 4ce4fbc4c4344da38c81053f87cf6862: loading cluster ID for follower catalog manager: success
02:04:28.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:28.565089 29680 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 4ce4fbc4c4344da38c81053f87cf6862: acquiring CA information for follower catalog manager: success
02:04:28.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:28.590718 29680 catalog_manager.cc:1637] T 00000000000000000000000000000000 P 4ce4fbc4c4344da38c81053f87cf6862: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
02:04:28.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 17 02:04:28 dist-test-slave-32ct krb5kdc[28797](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781661868, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.28.1@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:04:28.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:28.623775 29145 init.cc:295] Successfully reacquired a new kerberos TGT
02:04:29.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:29.170140 29701 raft_consensus.cc:493] T 00000000000000000000000000000000 P 4ce4fbc4c4344da38c81053f87cf6862 [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:04:29.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:29.172291 29701 raft_consensus.cc:515] T 00000000000000000000000000000000 P 4ce4fbc4c4344da38c81053f87cf6862 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4ce4fbc4c4344da38c81053f87cf6862" member_type: VOTER last_known_addr { host: "127.28.28.62" port: 46845 } } peers { permanent_uuid: "2c6224a4528c494eb782e1c4e5a71288" member_type: VOTER last_known_addr { host: "127.28.28.61" port: 38085 } } peers { permanent_uuid: "698282ccaf55491badb00732fef7a918" member_type: VOTER last_known_addr { host: "127.28.28.60" port: 37309 } }
02:04:29.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:29.193460 29701 leader_election.cc:290] T 00000000000000000000000000000000 P 4ce4fbc4c4344da38c81053f87cf6862 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 2c6224a4528c494eb782e1c4e5a71288 (127.28.28.61:38085), 698282ccaf55491badb00732fef7a918 (127.28.28.60:37309)
02:04:29.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:29.196527 29608 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.28.28.61:38085: connect: Connection refused (error 111)
02:04:29.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:29.247983 29608 leader_election.cc:336] T 00000000000000000000000000000000 P 4ce4fbc4c4344da38c81053f87cf6862 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 2c6224a4528c494eb782e1c4e5a71288 (127.28.28.61:38085): Network error: Client connection negotiation failed: client connection to 127.28.28.61:38085: connect: Connection refused (error 111)
02:04:29.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:29.250624 29605 leader_election.cc:336] T 00000000000000000000000000000000 P 4ce4fbc4c4344da38c81053f87cf6862 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 698282ccaf55491badb00732fef7a918 (127.28.28.60:37309): Network error: Client connection negotiation failed: client connection to 127.28.28.60:37309: connect: Connection refused (error 111)
02:04:29.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:29.251179 29605 leader_election.cc:304] T 00000000000000000000000000000000 P 4ce4fbc4c4344da38c81053f87cf6862 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 4ce4fbc4c4344da38c81053f87cf6862; no voters: 2c6224a4528c494eb782e1c4e5a71288, 698282ccaf55491badb00732fef7a918
02:04:29.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:29.252439 29701 raft_consensus.cc:2749] T 00000000000000000000000000000000 P 4ce4fbc4c4344da38c81053f87cf6862 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
02:04:29.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:28.020035 29691 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:04:29.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:29.418789 29689 debug-util.cc:398] Leaking SignalData structure 0x7b080002ef00 after lost signal to thread 29658
02:04:29.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:29.997692 29689 kernel_stack_watchdog.cc:198] Thread 29658 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 393ms:
02:04:29.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
02:04:29.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
02:04:29.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
02:04:29.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
02:04:29.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:29.998170 29658 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.975s user 0.001s sys 0.000s
02:04:29.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:29.998530 29658 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.975s user 0.001s sys 0.000s
02:04:30.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:30.000741 29693 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:04:30.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:30.005435 29658 server_base.cc:1061] running on GCE node
02:04:30.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:30.006673 29658 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:04:30.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:30.009639 29658 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:04:30.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:30.011955 29658 hybrid_clock.cc:648] HybridClock initialized: now 1781661870011920 us; error 44 us; skew 500 ppm
02:04:30.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:30.012761 29658 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:04:30.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 17 02:04:30 dist-test-slave-32ct krb5kdc[28797](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781661870, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.28.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:04:30.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:30.019086 29658 init.cc:377] Logged in from keytab as kudu/127.28.28.61@KRBTEST.COM (short username kudu)
02:04:30.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:30.024720 29658 webserver.cc:533] Webserver started at http://127.28.28.61:34599/ using document root <none> and password file <none>
02:04:30.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:30.025853 29658 fs_manager.cc:362] Metadata directory not provided
02:04:30.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:30.026129 29658 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:04:30.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:30.036965 29658 fs_manager.cc:714] Time spent opening directory manager: real 0.007s user 0.003s sys 0.002s
02:04:30.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:30.046236 29717 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:04:30.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:30.047843 29658 fs_manager.cc:730] Time spent opening block manager: real 0.009s user 0.005s sys 0.004s
02:04:30.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:30.048215 29658 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/master-1/data,/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/master-1/wal
02:04:30.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2c6224a4528c494eb782e1c4e5a71288"
02:04:30.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-17 02:04:11 on dist-test-slave-32ct"
02:04:30.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:30.051090 29658 fs_report.cc:389] FS layout report
02:04:30.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:04:30.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/master-1/wal
02:04:30.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/master-1/wal
02:04:30.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/master-1/data/data
02:04:30.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:04:30.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:04:30.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:04:30.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:04:30.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:04:30.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:04:30.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:04:30.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:04:30.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:04:30.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:04:30.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:04:30.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:30.114318 29658 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:04:30.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:30.123951 29658 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:04:30.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:30.124480 29658 kserver.cc:163] Server-wide thread pool size limit: 3276
02:04:30.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:30.216935 29658 rpc_server.cc:307] RPC server started. Bound to: 127.28.28.61:38085
02:04:30.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:30.217085 29769 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.28.28.61:38085 every 8 connection(s)
02:04:30.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:30.220014 29658 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/master-1/data/info.pb
02:04:30.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:30.225582 28784 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskqQ5_Rr/build/tsan/bin/kudu as pid 29658
02:04:30.227 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:04:30.227 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.28.28.60:37309
02:04:30.227 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
02:04:30.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:30.229224 28784 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskqQ5_Rr/build/tsan/bin/kudu
02:04:30.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskqQ5_Rr/build/tsan/bin/kudu
02:04:30.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/master-2/wal
02:04:30.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/master-2/data
02:04:30.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:04:30.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:04:30.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:04:30.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:04:30.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:04:30.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:04:30.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/master-2/logs
02:04:30.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/master-2/data/info.pb
02:04:30.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:04:30.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:04:30.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:04:30.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:04:30.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:04:30.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:04:30.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:04:30.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:04:30.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:04:30.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:04:30.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:04:30.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:04:30.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.28.60:37309
02:04:30.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.28.60
02:04:30.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38177
02:04:30.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/krb5kdc/kudu.keytab
02:04:30.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.28.60
02:04:30.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:04:30.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
02:04:30.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
02:04:30.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
02:04:30.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.28.20:40741
02:04:30.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:04:30.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:04:30.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:04:30.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:04:30.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.28.62:46845,127.28.28.61:38085,127.28.28.60:37309
02:04:30.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:04:30.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35823/jwks.json
02:04:30.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
02:04:30.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
02:04:30.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:30.234824 29770 sys_catalog.cc:263] Verifying existing consensus state
02:04:30.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:30.240314 29770 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 2c6224a4528c494eb782e1c4e5a71288: Bootstrap starting.
02:04:30.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:30.291095 29770 log.cc:826] T 00000000000000000000000000000000 P 2c6224a4528c494eb782e1c4e5a71288: Log is configured to *not* fsync() on all Append() calls
02:04:30.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:30.309325 29770 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 2c6224a4528c494eb782e1c4e5a71288: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
02:04:30.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:30.310374 29770 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 2c6224a4528c494eb782e1c4e5a71288: Bootstrap complete.
02:04:30.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:30.348109 29770 raft_consensus.cc:359] T 00000000000000000000000000000000 P 2c6224a4528c494eb782e1c4e5a71288 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4ce4fbc4c4344da38c81053f87cf6862" member_type: VOTER last_known_addr { host: "127.28.28.62" port: 46845 } } peers { permanent_uuid: "2c6224a4528c494eb782e1c4e5a71288" member_type: VOTER last_known_addr { host: "127.28.28.61" port: 38085 } } peers { permanent_uuid: "698282ccaf55491badb00732fef7a918" member_type: VOTER last_known_addr { host: "127.28.28.60" port: 37309 } }
02:04:30.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:30.351866 29770 raft_consensus.cc:740] T 00000000000000000000000000000000 P 2c6224a4528c494eb782e1c4e5a71288 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 2c6224a4528c494eb782e1c4e5a71288, State: Initialized, Role: FOLLOWER
02:04:30.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:30.353085 29770 consensus_queue.cc:260] T 00000000000000000000000000000000 P 2c6224a4528c494eb782e1c4e5a71288 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4ce4fbc4c4344da38c81053f87cf6862" member_type: VOTER last_known_addr { host: "127.28.28.62" port: 46845 } } peers { permanent_uuid: "2c6224a4528c494eb782e1c4e5a71288" member_type: VOTER last_known_addr { host: "127.28.28.61" port: 38085 } } peers { permanent_uuid: "698282ccaf55491badb00732fef7a918" member_type: VOTER last_known_addr { host: "127.28.28.60" port: 37309 } }
02:04:30.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:30.418984 29776 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2c6224a4528c494eb782e1c4e5a71288 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4ce4fbc4c4344da38c81053f87cf6862" member_type: VOTER last_known_addr { host: "127.28.28.62" port: 46845 } } peers { permanent_uuid: "2c6224a4528c494eb782e1c4e5a71288" member_type: VOTER last_known_addr { host: "127.28.28.61" port: 38085 } } peers { permanent_uuid: "698282ccaf55491badb00732fef7a918" member_type: VOTER last_known_addr { host: "127.28.28.60" port: 37309 } } }
02:04:30.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:30.420251 29776 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2c6224a4528c494eb782e1c4e5a71288 [sys.catalog]: This master's current role is: FOLLOWER
02:04:30.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:30.420996 29770 sys_catalog.cc:565] T 00000000000000000000000000000000 P 2c6224a4528c494eb782e1c4e5a71288 [sys.catalog]: configured and running, proceeding with master startup.
02:04:30.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 17 02:04:30 dist-test-slave-32ct krb5kdc[28797](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781661868, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.28.1@KRBTEST.COM for kudu/127.28.28.61@KRBTEST.COM
02:04:30.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:30.464336 29560 heartbeater.cc:344] Connected to a master server at 127.28.28.61:38085
02:04:30.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:30.472470 29265 heartbeater.cc:344] Connected to a master server at 127.28.28.61:38085
02:04:30.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:30.577685 29791 catalog_manager.cc:1294] Loaded cluster ID: f94c5ca418a8467e9c05f9f73c940f49
02:04:30.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:30.578002 29791 catalog_manager.cc:1587] T 00000000000000000000000000000000 P 2c6224a4528c494eb782e1c4e5a71288: loading cluster ID for follower catalog manager: success
02:04:30.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:30.588119 29791 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 2c6224a4528c494eb782e1c4e5a71288: acquiring CA information for follower catalog manager: success
02:04:30.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:30.593868 29791 catalog_manager.cc:1637] T 00000000000000000000000000000000 P 2c6224a4528c494eb782e1c4e5a71288: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
02:04:30.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 17 02:04:30 dist-test-slave-32ct krb5kdc[28797](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781661870, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.28.61@KRBTEST.COM for kudu/127.28.28.62@KRBTEST.COM
02:04:30.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:30.651944 29658 master_runner.cc:187] Error getting master registration for 127.28.28.60:37309: Network error: Client connection negotiation failed: client connection to 127.28.28.60:37309: connect: Connection refused (error 111),
02:04:30.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:30.714660 29658 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
02:04:30.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:30.734535 29412 heartbeater.cc:344] Connected to a master server at 127.28.28.61:38085
02:04:30.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:30.740891 29735 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "6ad9f08d43c343e1b54840b7c9e52ac9" instance_seqno: 1781661861418128) as {username='kudu', principal='kudu/127.28.28.2@KRBTEST.COM'} at 127.28.28.2:38597; Asking this server to re-register.
02:04:30.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:30.743449 29412 heartbeater.cc:461] Registering TS with master...
02:04:30.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:30.747288 29735 ts_manager.cc:194] Registered new tserver with Master: 6ad9f08d43c343e1b54840b7c9e52ac9 (127.28.28.2:33169)
02:04:30.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:30.796960 29772 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
02:04:30.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:30.797676 29772 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:04:30.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:30.798517 29772 flags.cc:432] Enabled unsafe flag: --never_fsync=true
02:04:30.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:30.838959 29772 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
02:04:30.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:30.839278 29772 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
02:04:30.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:30.839797 29772 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
02:04:30.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:30.840179 29772 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
02:04:30.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:30.840538 29772 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
02:04:30.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:30.840905 29772 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
02:04:30.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:30.841174 29772 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:35823/jwks.json
02:04:30.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:30.917816 29772 master_runner.cc:387] Master server non-default flags:
02:04:30.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:04:30.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.28.20:40741
02:04:30.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:04:30.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:04:30.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/master-2/data
02:04:30.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/master-2/wal
02:04:30.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
02:04:30.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:04:30.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.28.62:46845,127.28.28.61:38085,127.28.28.60:37309
02:04:30.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/krb5kdc/kudu.keytab
02:04:30.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.28.60
02:04:30.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:04:30.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:04:30.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:04:30.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.28.60:37309
02:04:30.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:04:30.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:04:30.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:04:30.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35823/jwks.json
02:04:30.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:04:30.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
02:04:30.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
02:04:30.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:04:30.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:04:30.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/master-2/data/info.pb
02:04:30.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.28.60
02:04:30.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38177
02:04:30.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:04:30.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:04:30.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:04:30.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:04:30.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:04:30.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/master-2/logs
02:04:30.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:04:30.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:04:30.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:04:30.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:04:30.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:04:30.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 23b10557f461bc2be3d2c282748d8de96f9c46ec
02:04:30.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:04:30.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 17 Jun 2026 01:43:23 UTC on e7f111948823
02:04:30.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12431
02:04:30.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:04:30.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:30.919601 29772 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:04:30.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:30.921167 29772 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:04:30.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:30.948933 29802 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:04:31.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:31.062803 29808 raft_consensus.cc:493] T 00000000000000000000000000000000 P 4ce4fbc4c4344da38c81053f87cf6862 [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:04:31.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:31.063735 29808 raft_consensus.cc:515] T 00000000000000000000000000000000 P 4ce4fbc4c4344da38c81053f87cf6862 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4ce4fbc4c4344da38c81053f87cf6862" member_type: VOTER last_known_addr { host: "127.28.28.62" port: 46845 } } peers { permanent_uuid: "2c6224a4528c494eb782e1c4e5a71288" member_type: VOTER last_known_addr { host: "127.28.28.61" port: 38085 } } peers { permanent_uuid: "698282ccaf55491badb00732fef7a918" member_type: VOTER last_known_addr { host: "127.28.28.60" port: 37309 } }
02:04:31.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:31.074865 29808 leader_election.cc:290] T 00000000000000000000000000000000 P 4ce4fbc4c4344da38c81053f87cf6862 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 2c6224a4528c494eb782e1c4e5a71288 (127.28.28.61:38085), 698282ccaf55491badb00732fef7a918 (127.28.28.60:37309)
02:04:31.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:31.097920 29745 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "4ce4fbc4c4344da38c81053f87cf6862" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "2c6224a4528c494eb782e1c4e5a71288" is_pre_election: true
02:04:31.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:31.099140 29745 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 2c6224a4528c494eb782e1c4e5a71288 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 4ce4fbc4c4344da38c81053f87cf6862 in term 1.
02:04:31.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:31.101581 29608 leader_election.cc:304] T 00000000000000000000000000000000 P 4ce4fbc4c4344da38c81053f87cf6862 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 2c6224a4528c494eb782e1c4e5a71288, 4ce4fbc4c4344da38c81053f87cf6862; no voters:
02:04:31.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:31.102849 29808 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 4ce4fbc4c4344da38c81053f87cf6862 [term 1 FOLLOWER]: Leader pre-election won for term 2
02:04:31.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:31.113795 29808 raft_consensus.cc:493] T 00000000000000000000000000000000 P 4ce4fbc4c4344da38c81053f87cf6862 [term 1 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
02:04:31.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:31.114375 29808 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 4ce4fbc4c4344da38c81053f87cf6862 [term 1 FOLLOWER]: Advancing to term 2
02:04:31.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:31.130952 29605 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.RequestConsensusVote -> {remote=127.28.28.60:37309, user_credentials={real_user=slave}} blocked reactor thread for 46566.4us
02:04:31.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:31.134922 29605 leader_election.cc:336] T 00000000000000000000000000000000 P 4ce4fbc4c4344da38c81053f87cf6862 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 698282ccaf55491badb00732fef7a918 (127.28.28.60:37309): Network error: Client connection negotiation failed: client connection to 127.28.28.60:37309: connect: Connection refused (error 111)
02:04:31.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:31.138811 29808 raft_consensus.cc:515] T 00000000000000000000000000000000 P 4ce4fbc4c4344da38c81053f87cf6862 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4ce4fbc4c4344da38c81053f87cf6862" member_type: VOTER last_known_addr { host: "127.28.28.62" port: 46845 } } peers { permanent_uuid: "2c6224a4528c494eb782e1c4e5a71288" member_type: VOTER last_known_addr { host: "127.28.28.61" port: 38085 } } peers { permanent_uuid: "698282ccaf55491badb00732fef7a918" member_type: VOTER last_known_addr { host: "127.28.28.60" port: 37309 } }
02:04:31.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:31.143184 29745 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "4ce4fbc4c4344da38c81053f87cf6862" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "2c6224a4528c494eb782e1c4e5a71288"
02:04:31.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:31.144352 29745 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 2c6224a4528c494eb782e1c4e5a71288 [term 1 FOLLOWER]: Advancing to term 2
02:04:31.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:31.148128 29808 leader_election.cc:290] T 00000000000000000000000000000000 P 4ce4fbc4c4344da38c81053f87cf6862 [CANDIDATE]: Term 2 election: Requested vote from peers 2c6224a4528c494eb782e1c4e5a71288 (127.28.28.61:38085), 698282ccaf55491badb00732fef7a918 (127.28.28.60:37309)
02:04:31.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 17 02:04:31 dist-test-slave-32ct krb5kdc[28797](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781661871, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.28.2@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:04:31.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:31.158354 29294 init.cc:295] Successfully reacquired a new kerberos TGT
02:04:31.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:31.159545 29605 leader_election.cc:336] T 00000000000000000000000000000000 P 4ce4fbc4c4344da38c81053f87cf6862 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 698282ccaf55491badb00732fef7a918 (127.28.28.60:37309): Network error: Client connection negotiation failed: client connection to 127.28.28.60:37309: connect: Connection refused (error 111)
02:04:31.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:31.178046 29745 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 2c6224a4528c494eb782e1c4e5a71288 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 4ce4fbc4c4344da38c81053f87cf6862 in term 2.
02:04:31.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:31.180274 29608 leader_election.cc:304] T 00000000000000000000000000000000 P 4ce4fbc4c4344da38c81053f87cf6862 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 2c6224a4528c494eb782e1c4e5a71288, 4ce4fbc4c4344da38c81053f87cf6862; no voters: 698282ccaf55491badb00732fef7a918
02:04:31.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:31.181293 29808 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 4ce4fbc4c4344da38c81053f87cf6862 [term 2 FOLLOWER]: Leader election won for term 2
02:04:31.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:31.185403 29808 raft_consensus.cc:697] T 00000000000000000000000000000000 P 4ce4fbc4c4344da38c81053f87cf6862 [term 2 LEADER]: Becoming Leader. State: Replica: 4ce4fbc4c4344da38c81053f87cf6862, State: Running, Role: LEADER
02:04:31.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:31.186810 29808 consensus_queue.cc:237] T 00000000000000000000000000000000 P 4ce4fbc4c4344da38c81053f87cf6862 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4ce4fbc4c4344da38c81053f87cf6862" member_type: VOTER last_known_addr { host: "127.28.28.62" port: 46845 } } peers { permanent_uuid: "2c6224a4528c494eb782e1c4e5a71288" member_type: VOTER last_known_addr { host: "127.28.28.61" port: 38085 } } peers { permanent_uuid: "698282ccaf55491badb00732fef7a918" member_type: VOTER last_known_addr { host: "127.28.28.60" port: 37309 } }
02:04:31.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:31.208784 29812 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4ce4fbc4c4344da38c81053f87cf6862 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 4ce4fbc4c4344da38c81053f87cf6862. Latest consensus state: current_term: 2 leader_uuid: "4ce4fbc4c4344da38c81053f87cf6862" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4ce4fbc4c4344da38c81053f87cf6862" member_type: VOTER last_known_addr { host: "127.28.28.62" port: 46845 } } peers { permanent_uuid: "2c6224a4528c494eb782e1c4e5a71288" member_type: VOTER last_known_addr { host: "127.28.28.61" port: 38085 } } peers { permanent_uuid: "698282ccaf55491badb00732fef7a918" member_type: VOTER last_known_addr { host: "127.28.28.60" port: 37309 } } }
02:04:31.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:31.209928 29812 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4ce4fbc4c4344da38c81053f87cf6862 [sys.catalog]: This master's current role is: LEADER
02:04:31.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:31.238842 29814 catalog_manager.cc:1510] Loading table and tablet metadata into memory...
02:04:31.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:31.272961 29814 catalog_manager.cc:1519] Initializing Kudu cluster ID...
02:04:31.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:31.305531 29814 catalog_manager.cc:1294] Loaded cluster ID: f94c5ca418a8467e9c05f9f73c940f49
02:04:31.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:31.305827 29814 catalog_manager.cc:1530] Initializing Kudu internal certificate authority...
02:04:31.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:31.310595 29814 catalog_manager.cc:1539] Loading token signing keys...
02:04:31.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:31.331194 29814 catalog_manager.cc:6080] T 00000000000000000000000000000000 P 4ce4fbc4c4344da38c81053f87cf6862: Loaded TSK: 0
02:04:31.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:31.341601 29814 catalog_manager.cc:1549] Initializing in-progress tserver states...
02:04:31.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:31.470875 29735 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "2c49f65da14341358a43e3ad6e67541a" instance_seqno: 1781661863980528) as {username='kudu', principal='kudu/127.28.28.3@KRBTEST.COM'} at 127.28.28.3:48893; Asking this server to re-register.
02:04:31.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:31.472534 29560 heartbeater.cc:461] Registering TS with master...
02:04:31.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:31.475692 29735 ts_manager.cc:194] Registered new tserver with Master: 2c49f65da14341358a43e3ad6e67541a (127.28.28.3:45433)
02:04:31.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:31.478632 29735 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "232880c2803f4de0851f2c9578cf2128" instance_seqno: 1781661859014261) as {username='kudu', principal='kudu/127.28.28.1@KRBTEST.COM'} at 127.28.28.1:56241; Asking this server to re-register.
02:04:31.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:31.480216 29265 heartbeater.cc:461] Registering TS with master...
02:04:31.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:31.484695 29735 ts_manager.cc:194] Registered new tserver with Master: 232880c2803f4de0851f2c9578cf2128 (127.28.28.1:40193)
02:04:31.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:31.663364 29745 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 2c6224a4528c494eb782e1c4e5a71288 [term 2 FOLLOWER]: Refusing update from remote peer 4ce4fbc4c4344da38c81053f87cf6862: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 2 index: 5. (index mismatch)
02:04:31.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:31.665196 29812 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 4ce4fbc4c4344da38c81053f87cf6862 [LEADER]: Connected to new peer: Peer: permanent_uuid: "2c6224a4528c494eb782e1c4e5a71288" member_type: VOTER last_known_addr { host: "127.28.28.61" port: 38085 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
02:04:31.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:31.682595 29817 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2c6224a4528c494eb782e1c4e5a71288 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 4ce4fbc4c4344da38c81053f87cf6862. Latest consensus state: current_term: 2 leader_uuid: "4ce4fbc4c4344da38c81053f87cf6862" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4ce4fbc4c4344da38c81053f87cf6862" member_type: VOTER last_known_addr { host: "127.28.28.62" port: 46845 } } peers { permanent_uuid: "2c6224a4528c494eb782e1c4e5a71288" member_type: VOTER last_known_addr { host: "127.28.28.61" port: 38085 } } peers { permanent_uuid: "698282ccaf55491badb00732fef7a918" member_type: VOTER last_known_addr { host: "127.28.28.60" port: 37309 } } }
02:04:31.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:31.683324 29817 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2c6224a4528c494eb782e1c4e5a71288 [sys.catalog]: This master's current role is: FOLLOWER
02:04:31.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:31.694327 29817 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2c6224a4528c494eb782e1c4e5a71288 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "4ce4fbc4c4344da38c81053f87cf6862" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4ce4fbc4c4344da38c81053f87cf6862" member_type: VOTER last_known_addr { host: "127.28.28.62" port: 46845 } } peers { permanent_uuid: "2c6224a4528c494eb782e1c4e5a71288" member_type: VOTER last_known_addr { host: "127.28.28.61" port: 38085 } } peers { permanent_uuid: "698282ccaf55491badb00732fef7a918" member_type: VOTER last_known_addr { host: "127.28.28.60" port: 37309 } } }
02:04:31.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:31.695030 29817 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2c6224a4528c494eb782e1c4e5a71288 [sys.catalog]: This master's current role is: FOLLOWER
02:04:31.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:31.707266 29808 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4ce4fbc4c4344da38c81053f87cf6862 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "4ce4fbc4c4344da38c81053f87cf6862" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4ce4fbc4c4344da38c81053f87cf6862" member_type: VOTER last_known_addr { host: "127.28.28.62" port: 46845 } } peers { permanent_uuid: "2c6224a4528c494eb782e1c4e5a71288" member_type: VOTER last_known_addr { host: "127.28.28.61" port: 38085 } } peers { permanent_uuid: "698282ccaf55491badb00732fef7a918" member_type: VOTER last_known_addr { host: "127.28.28.60" port: 37309 } } }
02:04:31.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:31.708201 29808 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4ce4fbc4c4344da38c81053f87cf6862 [sys.catalog]: This master's current role is: LEADER
02:04:31.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:31.739955 29605 consensus_peers.cc:597] T 00000000000000000000000000000000 P 4ce4fbc4c4344da38c81053f87cf6862 -> Peer 698282ccaf55491badb00732fef7a918 (127.28.28.60:37309): Couldn't send request to peer 698282ccaf55491badb00732fef7a918. Status: Network error: Client connection negotiation failed: client connection to 127.28.28.60:37309: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
02:04:32.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:32.407202 29561 heartbeater.cc:499] Master 127.28.28.62:46845 was elected leader, sending a full tablet report...
02:04:32.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:32.438858 29266 heartbeater.cc:499] Master 127.28.28.62:46845 was elected leader, sending a full tablet report...
02:04:32.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:32.678996 29573 master_runner.cc:187] Error getting master registration for 127.28.28.60:37309: Network error: Client connection negotiation failed: client connection to 127.28.28.60:37309: connect: Connection refused (error 111),
02:04:32.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:32.679414 29573 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
02:04:32.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:32.737028 29413 heartbeater.cc:499] Master 127.28.28.62:46845 was elected leader, sending a full tablet report...
02:04:33.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:32.349979 29801 debug-util.cc:398] Leaking SignalData structure 0x7b080002ef00 after lost signal to thread 29772
02:04:33.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:33.129099 29804 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection timed out after 2173 milliseconds
02:04:33.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:33.129379 29772 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 2.174s user 0.002s sys 0.000s
02:04:33.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:33.130450 29772 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 2.175s user 0.002s sys 0.001s
02:04:33.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:30.954445 29803 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:04:33.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:33.131596 29772 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:04:33.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:33.132210 29806 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:04:33.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:33.137202 29772 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:04:33.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:33.140278 29772 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:04:33.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:33.141705 29772 hybrid_clock.cc:648] HybridClock initialized: now 1781661873141761 us; error 141 us; skew 500 ppm
02:04:33.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:33.142972 29772 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:04:33.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 17 02:04:33 dist-test-slave-32ct krb5kdc[28797](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781661873, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.28.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:04:33.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:33.151026 29772 init.cc:377] Logged in from keytab as kudu/127.28.28.60@KRBTEST.COM (short username kudu)
02:04:33.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:33.183991 29772 webserver.cc:533] Webserver started at http://127.28.28.60:38177/ using document root <none> and password file <none>
02:04:33.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:33.185211 29772 fs_manager.cc:362] Metadata directory not provided
02:04:33.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:33.185544 29772 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:04:33.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:33.196419 29772 fs_manager.cc:714] Time spent opening directory manager: real 0.006s user 0.005s sys 0.000s
02:04:33.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:33.202282 29835 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:04:33.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:33.203480 29772 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.004s sys 0.000s
02:04:33.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:33.203751 29772 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/master-2/data,/tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/master-2/wal
02:04:33.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "698282ccaf55491badb00732fef7a918"
02:04:33.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-17 02:04:13 on dist-test-slave-32ct"
02:04:33.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:33.205633 29772 fs_report.cc:389] FS layout report
02:04:33.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:04:33.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/master-2/wal
02:04:33.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/master-2/wal
02:04:33.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/master-2/data/data
02:04:33.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:04:33.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:04:33.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:04:33.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:04:33.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:04:33.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:04:33.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:04:33.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:04:33.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:04:33.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:04:33.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:04:33.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:33.274273 29772 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:04:33.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:33.280165 29772 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:04:33.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:33.280640 29772 kserver.cc:163] Server-wide thread pool size limit: 3276
02:04:33.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:33.423705 29772 rpc_server.cc:307] RPC server started. Bound to: 127.28.28.60:37309
02:04:33.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:33.423710 29887 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.28.28.60:37309 every 8 connection(s)
02:04:33.430 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:04:33.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:33.427990 29772 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskqQ5_Rr/test-tmp/mini-kudu-cluster11535617578961946957/master-2/data/info.pb
02:04:33.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:33.429978 28784 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskqQ5_Rr/build/tsan/bin/kudu as pid 29772
02:04:33.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 17 02:04:33 dist-test-slave-32ct krb5kdc[28797](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781661868, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.28.1@KRBTEST.COM for kudu/127.28.28.60@KRBTEST.COM
02:04:33.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:33.563308 29888 sys_catalog.cc:263] Verifying existing consensus state
02:04:33.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:33.579818 29888 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 698282ccaf55491badb00732fef7a918: Bootstrap starting.
02:04:33.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:33.580828 29262 heartbeater.cc:344] Connected to a master server at 127.28.28.60:37309
02:04:33.614 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.28.28.62:46845(127.28.28.62:46845)] connecting to peer
02:04:33.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 17 02:04:33 dist-test-slave-32ct krb5kdc[28797](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781661873, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.28.3@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:04:33.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:33.639029 29441 init.cc:295] Successfully reacquired a new kerberos TGT
02:04:33.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:33.671197 29888 log.cc:826] T 00000000000000000000000000000000 P 698282ccaf55491badb00732fef7a918: Log is configured to *not* fsync() on all Append() calls
02:04:33.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 17 02:04:33 dist-test-slave-32ct krb5kdc[28797](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781661866, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.28.62@KRBTEST.COM for kudu/127.28.28.60@KRBTEST.COM
02:04:33.704 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.28.28.61:38085(127.28.28.61:38085)] connecting to peer
02:04:33.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:33.709693 29888 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 698282ccaf55491badb00732fef7a918: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
02:04:33.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:33.710737 29888 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 698282ccaf55491badb00732fef7a918: Bootstrap complete.
02:04:33.711 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.28.28.60:37309(127.28.28.60:37309)] connecting to peer
02:04:33.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:33.744570 29888 raft_consensus.cc:359] T 00000000000000000000000000000000 P 698282ccaf55491badb00732fef7a918 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4ce4fbc4c4344da38c81053f87cf6862" member_type: VOTER last_known_addr { host: "127.28.28.62" port: 46845 } } peers { permanent_uuid: "2c6224a4528c494eb782e1c4e5a71288" member_type: VOTER last_known_addr { host: "127.28.28.61" port: 38085 } } peers { permanent_uuid: "698282ccaf55491badb00732fef7a918" member_type: VOTER last_known_addr { host: "127.28.28.60" port: 37309 } }
02:04:33.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:33.748086 29888 raft_consensus.cc:740] T 00000000000000000000000000000000 P 698282ccaf55491badb00732fef7a918 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 698282ccaf55491badb00732fef7a918, State: Initialized, Role: FOLLOWER
02:04:33.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:33.749219 29888 consensus_queue.cc:260] T 00000000000000000000000000000000 P 698282ccaf55491badb00732fef7a918 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4ce4fbc4c4344da38c81053f87cf6862" member_type: VOTER last_known_addr { host: "127.28.28.62" port: 46845 } } peers { permanent_uuid: "2c6224a4528c494eb782e1c4e5a71288" member_type: VOTER last_known_addr { host: "127.28.28.61" port: 38085 } } peers { permanent_uuid: "698282ccaf55491badb00732fef7a918" member_type: VOTER last_known_addr { host: "127.28.28.60" port: 37309 } }
02:04:33.759 [DEBUG - kudu-nio-2] (Connection.java:757) [peer master-127.28.28.60:37309(127.28.28.60:37309)] Successfully connected to peer
02:04:33.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:33.758905 29898 sys_catalog.cc:455] T 00000000000000000000000000000000 P 698282ccaf55491badb00732fef7a918 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4ce4fbc4c4344da38c81053f87cf6862" member_type: VOTER last_known_addr { host: "127.28.28.62" port: 46845 } } peers { permanent_uuid: "2c6224a4528c494eb782e1c4e5a71288" member_type: VOTER last_known_addr { host: "127.28.28.61" port: 38085 } } peers { permanent_uuid: "698282ccaf55491badb00732fef7a918" member_type: VOTER last_known_addr { host: "127.28.28.60" port: 37309 } } }
02:04:33.763 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.28.28.61:38085(127.28.28.61:38085)] Successfully connected to peer
02:04:33.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:33.762645 29888 sys_catalog.cc:565] T 00000000000000000000000000000000 P 698282ccaf55491badb00732fef7a918 [sys.catalog]: configured and running, proceeding with master startup.
02:04:33.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:33.763427 29863 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 698282ccaf55491badb00732fef7a918 [term 1 FOLLOWER]: Advancing to term 2
02:04:33.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:33.759680 29898 sys_catalog.cc:458] T 00000000000000000000000000000000 P 698282ccaf55491badb00732fef7a918 [sys.catalog]: This master's current role is: FOLLOWER
02:04:33.769 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.28.28.62:46845(127.28.28.62:46845)] Successfully connected to peer
02:04:33.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 17 02:04:33 dist-test-slave-32ct krb5kdc[28797](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781661871, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.28.2@KRBTEST.COM for kudu/127.28.28.60@KRBTEST.COM
02:04:33.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:33.923060 29411 heartbeater.cc:344] Connected to a master server at 127.28.28.60:37309
02:04:33.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:33.928685 29898 sys_catalog.cc:455] T 00000000000000000000000000000000 P 698282ccaf55491badb00732fef7a918 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 4ce4fbc4c4344da38c81053f87cf6862. Latest consensus state: current_term: 2 leader_uuid: "4ce4fbc4c4344da38c81053f87cf6862" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4ce4fbc4c4344da38c81053f87cf6862" member_type: VOTER last_known_addr { host: "127.28.28.62" port: 46845 } } peers { permanent_uuid: "2c6224a4528c494eb782e1c4e5a71288" member_type: VOTER last_known_addr { host: "127.28.28.61" port: 38085 } } peers { permanent_uuid: "698282ccaf55491badb00732fef7a918" member_type: VOTER last_known_addr { host: "127.28.28.60" port: 37309 } } }
02:04:33.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:33.929548 29898 sys_catalog.cc:458] T 00000000000000000000000000000000 P 698282ccaf55491badb00732fef7a918 [sys.catalog]: This master's current role is: FOLLOWER
02:04:33.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:33.930850 29913 sys_catalog.cc:455] T 00000000000000000000000000000000 P 698282ccaf55491badb00732fef7a918 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "4ce4fbc4c4344da38c81053f87cf6862" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4ce4fbc4c4344da38c81053f87cf6862" member_type: VOTER last_known_addr { host: "127.28.28.62" port: 46845 } } peers { permanent_uuid: "2c6224a4528c494eb782e1c4e5a71288" member_type: VOTER last_known_addr { host: "127.28.28.61" port: 38085 } } peers { permanent_uuid: "698282ccaf55491badb00732fef7a918" member_type: VOTER last_known_addr { host: "127.28.28.60" port: 37309 } } }
02:04:33.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:33.931573 29913 sys_catalog.cc:458] T 00000000000000000000000000000000 P 698282ccaf55491badb00732fef7a918 [sys.catalog]: This master's current role is: FOLLOWER
02:04:33.944 [WARN - kudu-nio-1] (SecurityContext.java:206) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
02:04:33.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:33.962013 29917 catalog_manager.cc:1294] Loaded cluster ID: f94c5ca418a8467e9c05f9f73c940f49
02:04:33.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:33.962296 29917 catalog_manager.cc:1587] T 00000000000000000000000000000000 P 698282ccaf55491badb00732fef7a918: loading cluster ID for follower catalog manager: success
02:04:33.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:33.977980 29917 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 698282ccaf55491badb00732fef7a918: acquiring CA information for follower catalog manager: success
02:04:33.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:33.985440 29917 catalog_manager.cc:1637] T 00000000000000000000000000000000 P 698282ccaf55491badb00732fef7a918: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
02:04:33.987 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.28.28.61
02:04:33.987 [DEBUG - kudu-nio-2] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.28.28.60
02:04:33.987 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.28.28.62
02:04:33.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 17 02:04:33 dist-test-slave-32ct krb5kdc[28797](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781661873, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.28.60@KRBTEST.COM for kudu/127.28.28.62@KRBTEST.COM
02:04:34.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:34.183904 29812 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4ce4fbc4c4344da38c81053f87cf6862 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "4ce4fbc4c4344da38c81053f87cf6862" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4ce4fbc4c4344da38c81053f87cf6862" member_type: VOTER last_known_addr { host: "127.28.28.62" port: 46845 } } peers { permanent_uuid: "2c6224a4528c494eb782e1c4e5a71288" member_type: VOTER last_known_addr { host: "127.28.28.61" port: 38085 } } peers { permanent_uuid: "698282ccaf55491badb00732fef7a918" member_type: VOTER last_known_addr { host: "127.28.28.60" port: 37309 } } }
02:04:34.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:34.184948 29812 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4ce4fbc4c4344da38c81053f87cf6862 [sys.catalog]: This master's current role is: LEADER
02:04:34.342 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.28.28.60:37309(127.28.28.60:37309)] cleaning up while in state NEGOTIATING due to: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
02:04:34.343 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.28.28.60:37309: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
02:04:34.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:34.348134 29892 negotiation.cc:336] Failed RPC negotiation. Trace:
02:04:34.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0617 02:04:33.755613 (+ 0us) reactor.cc:730] Submitting negotiation task for server connection from 127.0.0.1:48896 (local address 127.28.28.60:37309)
02:04:34.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0617 02:04:33.756446 (+ 833us) server_negotiation.cc:207] Beginning negotiation
02:04:34.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0617 02:04:33.756483 (+ 37us) server_negotiation.cc:400] Waiting for connection header
02:04:34.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0617 02:04:33.880355 (+123872us) server_negotiation.cc:408] Connection header received
02:04:34.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0617 02:04:33.910171 (+ 29816us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
02:04:34.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0617 02:04:33.910218 (+ 47us) server_negotiation.cc:462] Received NEGOTIATE request from client
02:04:34.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0617 02:04:33.910463 (+ 245us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
02:04:34.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0617 02:04:33.910646 (+ 183us) server_negotiation.cc:227] Negotiated authn=SASL
02:04:34.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0617 02:04:34.163595 (+252949us) server_negotiation.cc:366] Received TLS_HANDSHAKE NegotiatePB request
02:04:34.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0617 02:04:34.164894 (+ 1299us) server_negotiation.cc:378] Sending TLS_HANDSHAKE NegotiatePB response
02:04:34.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0617 02:04:34.323877 (+158983us) server_negotiation.cc:366] Received TLS_HANDSHAKE NegotiatePB request
02:04:34.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0617 02:04:34.324492 (+ 615us) server_negotiation.cc:663] Negotiated TLSv1.3 with cipher TLS_AES_128_GCM_SHA256 TLSv1.3 Kx=any Au=any Enc=AESGCM(128) Mac=AEAD
02:04:34.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0617 02:04:34.346217 (+ 21725us) negotiation.cc:326] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:48896: BlockingRecv error: failed to read from TLS socket (remote: 127.0.0.1:48896): Cannot send after transport endpoint shutdown (error 108)
02:04:34.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":133}
02:04:34.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:34.350550 29559 heartbeater.cc:344] Connected to a master server at 127.28.28.60:37309
02:04:34.355 [DEBUG - kudu-nio-2] (Connection.java:239) [peer master-127.28.28.60:37309(127.28.28.60:37309)] handling channelInactive
02:04:34.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:34.356649 29852 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "2c49f65da14341358a43e3ad6e67541a" instance_seqno: 1781661863980528) as {username='kudu', principal='kudu/127.28.28.3@KRBTEST.COM'} at 127.28.28.3:59947; Asking this server to re-register.
02:04:34.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:34.361074 29559 heartbeater.cc:461] Registering TS with master...
02:04:34.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:34.364317 29852 ts_manager.cc:194] Registered new tserver with Master: 2c49f65da14341358a43e3ad6e67541a (127.28.28.3:45433)
02:04:34.365 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.28.28.62:46845(127.28.28.62:46845)] cleaning up while in state NEGOTIATING due to: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
02:04:34.365 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.28.28.62:46845: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
02:04:34.366 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.28.28.62:46845(127.28.28.62:46845)] handling channelInactive
02:04:34.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:34.369099 29903 negotiation.cc:336] Failed RPC negotiation. Trace:
02:04:34.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0617 02:04:33.768818 (+ 0us) reactor.cc:730] Submitting negotiation task for server connection from 127.0.0.1:57968 (local address 127.28.28.62:46845)
02:04:34.369 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.28.28.61:38085(127.28.28.61:38085)] cleaning up while in state NEGOTIATING due to: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
02:04:34.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0617 02:04:33.785367 (+ 16549us) server_negotiation.cc:207] Beginning negotiation
02:04:34.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0617 02:04:33.785395 (+ 28us) server_negotiation.cc:400] Waiting for connection header
02:04:34.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0617 02:04:33.880007 (+ 94612us) server_negotiation.cc:408] Connection header received
02:04:34.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0617 02:04:33.920892 (+ 40885us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
02:04:34.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0617 02:04:33.920927 (+ 35us) server_negotiation.cc:462] Received NEGOTIATE request from client
02:04:34.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0617 02:04:33.921155 (+ 228us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
02:04:34.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0617 02:04:33.921372 (+ 217us) server_negotiation.cc:227] Negotiated authn=SASL
02:04:34.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0617 02:04:34.170140 (+248768us) server_negotiation.cc:366] Received TLS_HANDSHAKE NegotiatePB request
02:04:34.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0617 02:04:34.171784 (+ 1644us) server_negotiation.cc:378] Sending TLS_HANDSHAKE NegotiatePB response
02:04:34.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0617 02:04:34.323879 (+152095us) server_negotiation.cc:366] Received TLS_HANDSHAKE NegotiatePB request
02:04:34.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0617 02:04:34.324641 (+ 762us) server_negotiation.cc:663] Negotiated TLSv1.3 with cipher TLS_AES_128_GCM_SHA256 TLSv1.3 Kx=any Au=any Enc=AESGCM(128) Mac=AEAD
02:04:34.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0617 02:04:34.367401 (+ 42760us) negotiation.cc:326] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:57968: BlockingRecv error: failed to read from TLS socket (remote: 127.0.0.1:57968): Cannot send after transport endpoint shutdown (error 108)
02:04:34.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":15685,"thread_start_us":16174,"threads_started":1}
02:04:34.370 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.28.28.61:38085: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
02:04:34.377 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1523) Notify RPC KuduRpc(method=ListTabletServers, tablet=null, attempt=1, TimeoutTracker(timeout=30000, elapsed=942), Traces: [0ms] refreshing cache from master, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.28.62:46845, [150ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.28.61:38085, [158ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.28.60:37309, [790ms] Sub RPC ConnectToMaster: received response from server master-127.28.28.60:37309: Configuration error: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), [812ms] Sub RPC ConnectToMaster: received response from server master-127.28.28.62:46845: Configuration error: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), [817ms] Sub RPC ConnectToMaster: received response from server master-127.28.28.61:38085: Configuration error: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), deferred=null) after lookup exception
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.28.28.62:46845,127.28.28.61:38085,127.28.28.60:37309). Exceptions received: [org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
at java.base/java.lang.Thread.run(Thread.java:840)
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.28.28.62:46845,127.28.28.61:38085,127.28.28.60:37309). Exceptions received: [org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
at java.base/java.lang.Thread.run(Thread.java:840)
02:04:34.392 [DEBUG - kudu-nio-1] (KuduRpc.java:260) Handling a callback on RPC KuduRpc(method=ListTabletServers, tablet=null, attempt=1, TimeoutTracker(timeout=30000, elapsed=957), Traces: [0ms] refreshing cache from master, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.28.62:46845, [150ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.28.61:38085, [158ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.28.60:37309, [790ms] Sub RPC ConnectToMaster: received response from server master-127.28.28.60:37309: Configuration error: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), [812ms] Sub RPC ConnectToMaster: received response from server master-127.28.28.62:46845: Configuration error: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), [817ms] Sub RPC ConnectToMaster: received response from server master-127.28.28.61:38085: Configuration error: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), deferred=null) with no deferred attached!
02:04:34.395 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.28.28.60:37309(127.28.28.60:37309)] disconnecting while in state TERMINATED
02:04:34.396 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.28.28.62:46845(127.28.28.62:46845)] disconnecting while in state TERMINATED
02:04:34.396 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.28.28.61:38085(127.28.28.61:38085)] disconnecting while in state TERMINATED
02:04:34.417 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2704) Releasing all remaining resources
02:04:34.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:34.419255 29899 negotiation.cc:336] Failed RPC negotiation. Trace:
02:04:34.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0617 02:04:33.754408 (+ 0us) reactor.cc:730] Submitting negotiation task for server connection from 127.0.0.1:49750 (local address 127.28.28.61:38085)
02:04:34.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0617 02:04:33.856447 (+102039us) server_negotiation.cc:207] Beginning negotiation
02:04:34.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0617 02:04:33.856469 (+ 22us) server_negotiation.cc:400] Waiting for connection header
02:04:34.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0617 02:04:33.882145 (+ 25676us) server_negotiation.cc:408] Connection header received
02:04:34.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0617 02:04:33.921506 (+ 39361us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
02:04:34.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0617 02:04:33.921571 (+ 65us) server_negotiation.cc:462] Received NEGOTIATE request from client
02:04:34.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0617 02:04:33.921857 (+ 286us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
02:04:34.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0617 02:04:33.922065 (+ 208us) server_negotiation.cc:227] Negotiated authn=SASL
02:04:34.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0617 02:04:34.159535 (+237470us) server_negotiation.cc:366] Received TLS_HANDSHAKE NegotiatePB request
02:04:34.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0617 02:04:34.161439 (+ 1904us) server_negotiation.cc:378] Sending TLS_HANDSHAKE NegotiatePB response
02:04:34.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0617 02:04:34.328124 (+166685us) server_negotiation.cc:366] Received TLS_HANDSHAKE NegotiatePB request
02:04:34.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0617 02:04:34.328752 (+ 628us) server_negotiation.cc:663] Negotiated TLSv1.3 with cipher TLS_AES_128_GCM_SHA256 TLSv1.3 Kx=any Au=any Enc=AESGCM(128) Mac=AEAD
02:04:34.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0617 02:04:34.417467 (+ 88715us) negotiation.cc:326] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:49750: BlockingRecv error: failed to read from TLS socket (remote: 127.0.0.1:49750): Cannot send after transport endpoint shutdown (error 108)
02:04:34.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":101520,"thread_start_us":101994,"threads_started":1}
02:04:34.428 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
02:04:34.429 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.28.28.61:38085(127.28.28.61:38085)] handling channelInactive
02:04:34.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:34.458573 28784 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskqQ5_Rr/build/tsan/bin/kudu with pid 29126
02:04:34.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:34.459769 28488 debug-util.cc:398] Leaking SignalData structure 0x7b08000b55a0 after lost signal to thread 28362
02:04:34.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 02:04:34.461007 28488 debug-util.cc:398] Leaking SignalData structure 0x7b08000ad260 after lost signal to thread 28491
02:04:34.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:34.492498 28784 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskqQ5_Rr/build/tsan/bin/kudu with pid 29276
02:04:34.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:34.535866 28784 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskqQ5_Rr/build/tsan/bin/kudu with pid 29425
02:04:34.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:34.598819 28784 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskqQ5_Rr/build/tsan/bin/kudu with pid 29573
02:04:34.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:34.678402 28784 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskqQ5_Rr/build/tsan/bin/kudu with pid 29658
02:04:34.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:34.727222 28784 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskqQ5_Rr/build/tsan/bin/kudu with pid 29772
02:04:35.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-17T02:04:35Z chronyd exiting
02:04:37.020 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
02:04:37.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:37.023834 28059 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskqQ5_Rr/build/tsan/bin/kudu with pid 28361
02:04:37.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:37.051688 28059 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskqQ5_Rr/build/tsan/bin/kudu with pid 28502
02:04:37.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:37.078707 28059 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskqQ5_Rr/build/tsan/bin/kudu with pid 28642
02:04:37.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:37.105876 28059 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskqQ5_Rr/build/tsan/bin/kudu with pid 28071
02:04:37.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:37.135202 28059 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskqQ5_Rr/build/tsan/bin/kudu with pid 28141
02:04:37.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 02:04:37.165599 28059 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskqQ5_Rr/build/tsan/bin/kudu with pid 28216
02:04:38.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-17T02:04:38Z chronyd exiting
02:04:39.354 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.28.28.62:46845,127.28.28.61:38085,127.28.28.60:37309). Exceptions received: [org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)]
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:569)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at java.base/java.lang.Thread.run(Thread.java:840)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
... 1 more
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.28.28.62:46845,127.28.28.61:38085,127.28.28.60:37309). Exceptions received: [org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)]
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:569)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at java.base/java.lang.Thread.run(Thread.java:840)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
... 1 more
02:04:39.756 [ERROR - main] (RetryRule.java:253) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: giving up after 1 attempts
02:04:39.756 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.28.28.62:46845,127.28.28.61:38085,127.28.28.60:37309). Exceptions received: [org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)]
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:569)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at java.base/java.lang.Thread.run(Thread.java:840)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
... 1 more
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.28.28.62:46845,127.28.28.61:38085,127.28.28.60:37309). Exceptions received: [org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)]
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:569)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at java.base/java.lang.Thread.run(Thread.java:840)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
... 1 more