20:03:27.984 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
20:03:27.997 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskqmogBp/test-tmp
20:03:28.002 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
20:03:28.006 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskqmogBp/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
20:03:28.261 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster11212040801989529672"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
20:03:28.448 [DEBUG - main] (MiniKuduCluster.java:195) Response:
20:03:28.450 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
20:03:28.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:28.454277 22001 mini_oidc.cc:150] Starting JWKS server
20:03:28.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:28.457866 22001 webserver.cc:492] Webserver started at http://127.0.0.1:46041/ using document root <none> and password file <none>
20:03:28.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:28.458519 22001 mini_oidc.cc:186] Starting OIDC Discovery server
20:03:28.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:28.460963 22001 webserver.cc:492] Webserver started at http://127.0.0.1:39067/ using document root <none> and password file <none>
20:03:28.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-05-03T20:03:28Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
20:03:28.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-05-03T20:03:28Z Disabled control of system clock
20:03:28.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:28.525302 22001 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskqmogBp/build/tsan/bin/kudu
20:03:28.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskqmogBp/build/tsan/bin/kudu
20:03:28.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster11212040801989529672/master-0/wal
20:03:28.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster11212040801989529672/master-0/data
20:03:28.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:03:28.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:03:28.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:03:28.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:03:28.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:03:28.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:03:28.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster11212040801989529672/master-0/logs
20:03:28.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster11212040801989529672/master-0/data/info.pb
20:03:28.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:03:28.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:03:28.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:03:28.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:03:28.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:03:28.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:03:28.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:03:28.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:03:28.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:03:28.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:03:28.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:03:28.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:03:28.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.124.126:41563
20:03:28.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.124.126
20:03:28.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:03:28.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.124.84:42337
20:03:28.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:03:28.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:03:28.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:03:28.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:03:28.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.21.124.126:41563,127.21.124.125:46783,127.21.124.124:45763
20:03:28.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:03:28.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46041/jwks.json with env {}
20:03:28.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:28.835544 22013 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:03:28.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:28.836117 22013 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:03:28.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:28.836707 22013 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:03:28.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:28.870801 22013 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
20:03:28.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:28.871112 22013 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:03:28.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:28.871306 22013 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
20:03:28.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:28.871488 22013 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
20:03:28.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:28.871766 22013 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
20:03:28.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:28.872064 22013 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:46041/jwks.json
20:03:28.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:28.910658 22013 master_runner.cc:386] Master server non-default flags:
20:03:28.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:03:28.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.124.84:42337
20:03:28.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:03:28.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:03:28.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster11212040801989529672/master-0/data
20:03:28.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster11212040801989529672/master-0/wal
20:03:28.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:03:28.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.21.124.126:41563,127.21.124.125:46783,127.21.124.124:45763
20:03:28.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:03:28.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:03:28.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:03:28.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.124.126:41563
20:03:28.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:03:28.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:03:28.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:03:28.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46041/jwks.json
20:03:28.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:03:28.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:03:28.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster11212040801989529672/master-0/data/info.pb
20:03:28.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.124.126
20:03:28.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:03:28.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:03:28.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:03:28.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:03:28.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:03:28.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:03:28.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster11212040801989529672/master-0/logs
20:03:28.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:03:28.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:03:28.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:03:28.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:03:28.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:03:28.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0ba7f54e15c9f673bf3e350f30b8d5f1a4037b79
20:03:28.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:03:28.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 03 May 2026 19:43:25 UTC on e7f111948823
20:03:28.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11731
20:03:28.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:03:28.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:28.912052 22013 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:03:28.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:28.914309 22013 file_cache.cc:492] Constructed file cache file cache with capacity 419430
20:03:28.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:28.929317 22019 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:30.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:28.931245 22020 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:30.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:30.193054 22022 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:30.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:30.196661 22021 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1260 milliseconds
20:03:30.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:30.196748 22013 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:03:30.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:30.198081 22013 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:03:30.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:30.200727 22013 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:03:30.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:30.202203 22013 hybrid_clock.cc:648] HybridClock initialized: now 1777838610202157 us; error 53 us; skew 500 ppm
20:03:30.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:30.203068 22013 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:03:30.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:30.210819 22013 webserver.cc:492] Webserver started at http://127.21.124.126:34911/ using document root <none> and password file <none>
20:03:30.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:30.212020 22013 fs_manager.cc:362] Metadata directory not provided
20:03:30.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:30.212244 22013 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:03:30.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:30.212736 22013 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
20:03:30.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:30.217358 22013 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster11212040801989529672/master-0/data/instance:
20:03:30.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c87e8c6d5a524a2a84f403153fd2287b"
20:03:30.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-03 20:03:30 on dist-test-slave-b0rl"
20:03:30.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:30.218554 22013 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster11212040801989529672/master-0/wal/instance:
20:03:30.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c87e8c6d5a524a2a84f403153fd2287b"
20:03:30.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-03 20:03:30 on dist-test-slave-b0rl"
20:03:30.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:30.226502 22013 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.006s sys 0.001s
20:03:30.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:30.232440 22029 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:03:30.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:30.234017 22013 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
20:03:30.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:30.234438 22013 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster11212040801989529672/master-0/data,/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster11212040801989529672/master-0/wal
20:03:30.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c87e8c6d5a524a2a84f403153fd2287b"
20:03:30.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-03 20:03:30 on dist-test-slave-b0rl"
20:03:30.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:30.234968 22013 fs_report.cc:389] FS layout report
20:03:30.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:03:30.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster11212040801989529672/master-0/wal
20:03:30.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster11212040801989529672/master-0/wal
20:03:30.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster11212040801989529672/master-0/data/data
20:03:30.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:03:30.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:03:30.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:03:30.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:03:30.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:03:30.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:03:30.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:03:30.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:03:30.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:03:30.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:03:30.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:03:30.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:30.303638 22013 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:03:30.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:30.305450 22013 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:03:30.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:30.305997 22013 kserver.cc:163] Server-wide thread pool size limit: 3276
20:03:30.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:30.381156 22013 rpc_server.cc:307] RPC server started. Bound to: 127.21.124.126:41563
20:03:30.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:30.381234 22080 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.21.124.126:41563 every 8 connection(s)
20:03:30.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:30.384327 22013 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster11212040801989529672/master-0/data/info.pb
20:03:30.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:30.385425 22001 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskqmogBp/build/tsan/bin/kudu as pid 22013
20:03:30.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:30.386003 22001 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster11212040801989529672/master-0/wal/instance
20:03:30.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:30.390506 22001 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskqmogBp/build/tsan/bin/kudu
20:03:30.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskqmogBp/build/tsan/bin/kudu
20:03:30.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster11212040801989529672/master-1/wal
20:03:30.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster11212040801989529672/master-1/data
20:03:30.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:03:30.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:03:30.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:03:30.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:03:30.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:03:30.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:03:30.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster11212040801989529672/master-1/logs
20:03:30.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster11212040801989529672/master-1/data/info.pb
20:03:30.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:03:30.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:03:30.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:03:30.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:03:30.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:03:30.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:03:30.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:03:30.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:03:30.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:03:30.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:03:30.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:03:30.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:03:30.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.124.125:46783
20:03:30.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.124.125
20:03:30.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:03:30.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.124.84:42337
20:03:30.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:03:30.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:03:30.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:03:30.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:03:30.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.21.124.126:41563,127.21.124.125:46783,127.21.124.124:45763
20:03:30.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:03:30.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46041/jwks.json with env {}
20:03:30.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:30.391338 22081 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
20:03:30.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:30.407289 22081 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.21.124.126" port: 41563 } has no permanent_uuid. Determining permanent_uuid...
20:03:30.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:30.434784 22081 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.21.124.125" port: 46783 } has no permanent_uuid. Determining permanent_uuid...
20:03:30.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:30.437866 22032 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.21.124.125:46783: connect: Connection refused (error 111)
20:03:30.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:30.442184 22081 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.21.124.125:46783: Network error: Client connection negotiation failed: client connection to 127.21.124.125:46783: connect: Connection refused (error 111)
20:03:30.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:30.491719 22081 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.124.125" port: 46783 } attempt: 1
20:03:30.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:30.496475 22081 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.21.124.125:46783: Network error: Client connection negotiation failed: client connection to 127.21.124.125:46783: connect: Connection refused (error 111)
20:03:30.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:30.564932 22081 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.124.125" port: 46783 } attempt: 2
20:03:30.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:30.570662 22081 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.21.124.125:46783: Network error: Client connection negotiation failed: client connection to 127.21.124.125:46783: connect: Connection refused (error 111)
20:03:30.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:30.662135 22081 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.124.125" port: 46783 } attempt: 3
20:03:30.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:30.667097 22081 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.21.124.125:46783: Network error: Client connection negotiation failed: client connection to 127.21.124.125:46783: connect: Connection refused (error 111)
20:03:30.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:30.724445 22083 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:03:30.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:30.725167 22083 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:03:30.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:30.725921 22083 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:03:30.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:30.760465 22083 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
20:03:30.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:30.760870 22083 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:03:30.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:30.761201 22083 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
20:03:30.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:30.761543 22083 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
20:03:30.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:30.761924 22083 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
20:03:30.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:30.762215 22083 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:46041/jwks.json
20:03:30.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:30.800051 22083 master_runner.cc:386] Master server non-default flags:
20:03:30.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:03:30.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.124.84:42337
20:03:30.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:03:30.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:03:30.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster11212040801989529672/master-1/data
20:03:30.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster11212040801989529672/master-1/wal
20:03:30.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:03:30.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.21.124.126:41563,127.21.124.125:46783,127.21.124.124:45763
20:03:30.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:03:30.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:03:30.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:03:30.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.124.125:46783
20:03:30.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:03:30.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:03:30.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:03:30.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46041/jwks.json
20:03:30.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:03:30.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:03:30.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster11212040801989529672/master-1/data/info.pb
20:03:30.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.124.125
20:03:30.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:03:30.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:03:30.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:03:30.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:03:30.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:03:30.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:03:30.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster11212040801989529672/master-1/logs
20:03:30.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:03:30.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:03:30.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:03:30.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:03:30.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:03:30.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0ba7f54e15c9f673bf3e350f30b8d5f1a4037b79
20:03:30.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:03:30.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 03 May 2026 19:43:25 UTC on e7f111948823
20:03:30.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11731
20:03:30.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:03:30.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:30.801874 22083 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:03:30.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:30.804277 22083 file_cache.cc:492] Constructed file cache file cache with capacity 419430
20:03:30.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:30.810562 22081 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.124.125" port: 46783 } attempt: 4
20:03:30.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:30.816767 22081 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.21.124.125:46783: Network error: Client connection negotiation failed: client connection to 127.21.124.125:46783: connect: Connection refused (error 111)
20:03:30.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:30.818375 22092 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:31.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:31.116302 22081 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.124.125" port: 46783 } attempt: 5
20:03:31.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:31.123910 22081 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.21.124.125:46783: Network error: Client connection negotiation failed: client connection to 127.21.124.125:46783: connect: Connection refused (error 111)
20:03:31.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:31.671546 22081 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.124.125" port: 46783 } attempt: 6
20:03:31.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:31.682548 22081 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.21.124.125:46783: Network error: Client connection negotiation failed: client connection to 127.21.124.125:46783: connect: Connection refused (error 111)
20:03:32.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:30.819114 22093 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:32.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:32.101016 22083 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 1.284s user 0.453s sys 0.828s
20:03:32.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:32.102190 22083 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 1.286s user 0.455s sys 0.830s
20:03:32.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:32.104576 22083 server_base.cc:1061] running on GCE node
20:03:32.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:32.106313 22099 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:32.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:32.107699 22083 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:03:32.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:32.110423 22083 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:03:32.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:32.111976 22083 hybrid_clock.cc:648] HybridClock initialized: now 1777838612111926 us; error 59 us; skew 500 ppm
20:03:32.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:32.113044 22083 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:03:32.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:32.120464 22083 webserver.cc:492] Webserver started at http://127.21.124.125:38195/ using document root <none> and password file <none>
20:03:32.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:32.121547 22083 fs_manager.cc:362] Metadata directory not provided
20:03:32.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:32.121803 22083 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:03:32.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:32.122452 22083 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
20:03:32.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:32.127417 22083 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster11212040801989529672/master-1/data/instance:
20:03:32.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8130d13960a14c88b4095c433ca69b07"
20:03:32.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-03 20:03:32 on dist-test-slave-b0rl"
20:03:32.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:32.128839 22083 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster11212040801989529672/master-1/wal/instance:
20:03:32.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8130d13960a14c88b4095c433ca69b07"
20:03:32.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-03 20:03:32 on dist-test-slave-b0rl"
20:03:32.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:32.137429 22083 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.007s sys 0.003s
20:03:32.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:32.143858 22104 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:03:32.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:32.146440 22083 fs_manager.cc:730] Time spent opening block manager: real 0.006s user 0.000s sys 0.004s
20:03:32.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:32.146838 22083 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster11212040801989529672/master-1/data,/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster11212040801989529672/master-1/wal
20:03:32.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8130d13960a14c88b4095c433ca69b07"
20:03:32.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-03 20:03:32 on dist-test-slave-b0rl"
20:03:32.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:32.147305 22083 fs_report.cc:389] FS layout report
20:03:32.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:03:32.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster11212040801989529672/master-1/wal
20:03:32.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster11212040801989529672/master-1/wal
20:03:32.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster11212040801989529672/master-1/data/data
20:03:32.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:03:32.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:03:32.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:03:32.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:03:32.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:03:32.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:03:32.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:03:32.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:03:32.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:03:32.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:03:32.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:03:32.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:32.217550 22083 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:03:32.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:32.219264 22083 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:03:32.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:32.219779 22083 kserver.cc:163] Server-wide thread pool size limit: 3276
20:03:32.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:32.295205 22083 rpc_server.cc:307] RPC server started. Bound to: 127.21.124.125:46783
20:03:32.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:32.295274 22155 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.21.124.125:46783 every 8 connection(s)
20:03:32.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:32.298137 22083 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster11212040801989529672/master-1/data/info.pb
20:03:32.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:32.303220 22001 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskqmogBp/build/tsan/bin/kudu as pid 22083
20:03:32.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:32.303676 22001 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster11212040801989529672/master-1/wal/instance
20:03:32.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:32.304910 22156 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
20:03:32.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:32.306689 22001 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskqmogBp/build/tsan/bin/kudu
20:03:32.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskqmogBp/build/tsan/bin/kudu
20:03:32.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster11212040801989529672/master-2/wal
20:03:32.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster11212040801989529672/master-2/data
20:03:32.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:03:32.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:03:32.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:03:32.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:03:32.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:03:32.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:03:32.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster11212040801989529672/master-2/logs
20:03:32.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster11212040801989529672/master-2/data/info.pb
20:03:32.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:03:32.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:03:32.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:03:32.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:03:32.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:03:32.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:03:32.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:03:32.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:03:32.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:03:32.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:03:32.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:03:32.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:03:32.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.124.124:45763
20:03:32.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.124.124
20:03:32.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:03:32.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.124.84:42337
20:03:32.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:03:32.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:03:32.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:03:32.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:03:32.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.21.124.126:41563,127.21.124.125:46783,127.21.124.124:45763
20:03:32.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:03:32.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46041/jwks.json with env {}
20:03:32.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:32.318048 22156 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.21.124.126" port: 41563 } has no permanent_uuid. Determining permanent_uuid...
20:03:32.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:32.343456 22156 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.21.124.125" port: 46783 } has no permanent_uuid. Determining permanent_uuid...
20:03:32.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:32.358670 22156 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.21.124.124" port: 45763 } has no permanent_uuid. Determining permanent_uuid...
20:03:32.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:32.361157 22106 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.21.124.124:45763: connect: Connection refused (error 111)
20:03:32.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:32.365712 22156 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.21.124.124:45763: Network error: Client connection negotiation failed: client connection to 127.21.124.124:45763: connect: Connection refused (error 111)
20:03:32.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:32.415374 22156 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.124.124" port: 45763 } attempt: 1
20:03:32.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:32.420409 22156 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.21.124.124:45763: Network error: Client connection negotiation failed: client connection to 127.21.124.124:45763: connect: Connection refused (error 111)
20:03:32.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:32.488890 22156 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.124.124" port: 45763 } attempt: 2
20:03:32.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:32.493707 22156 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.21.124.124:45763: Network error: Client connection negotiation failed: client connection to 127.21.124.124:45763: connect: Connection refused (error 111)
20:03:32.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:32.585177 22156 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.124.124" port: 45763 } attempt: 3
20:03:32.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:32.589453 22156 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.21.124.124:45763: Network error: Client connection negotiation failed: client connection to 127.21.124.124:45763: connect: Connection refused (error 111)
20:03:32.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:32.633296 22158 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:03:32.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:32.633975 22158 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:03:32.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:32.634534 22158 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:03:32.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:32.667975 22158 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
20:03:32.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:32.668267 22158 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:03:32.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:32.668462 22158 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
20:03:32.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:32.668645 22158 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
20:03:32.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:32.668836 22158 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
20:03:32.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:32.668972 22158 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:46041/jwks.json
20:03:32.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:32.706367 22158 master_runner.cc:386] Master server non-default flags:
20:03:32.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:03:32.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.124.84:42337
20:03:32.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:03:32.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:03:32.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster11212040801989529672/master-2/data
20:03:32.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster11212040801989529672/master-2/wal
20:03:32.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:03:32.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.21.124.126:41563,127.21.124.125:46783,127.21.124.124:45763
20:03:32.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:03:32.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:03:32.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:03:32.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.124.124:45763
20:03:32.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:03:32.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:03:32.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:03:32.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46041/jwks.json
20:03:32.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:03:32.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:03:32.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster11212040801989529672/master-2/data/info.pb
20:03:32.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.124.124
20:03:32.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:03:32.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:03:32.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:03:32.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:03:32.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:03:32.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:03:32.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster11212040801989529672/master-2/logs
20:03:32.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:03:32.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:03:32.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:03:32.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:03:32.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:03:32.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0ba7f54e15c9f673bf3e350f30b8d5f1a4037b79
20:03:32.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:03:32.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 03 May 2026 19:43:25 UTC on e7f111948823
20:03:32.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11731
20:03:32.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:03:32.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:32.707844 22158 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:03:32.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:32.709846 22158 file_cache.cc:492] Constructed file cache file cache with capacity 419430
20:03:32.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:32.722347 22168 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:32.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:32.732951 22156 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.124.124" port: 45763 } attempt: 4
20:03:32.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:32.741353 22156 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.21.124.124:45763: Network error: Client connection negotiation failed: client connection to 127.21.124.124:45763: connect: Connection refused (error 111)
20:03:32.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:32.743156 22081 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.124.125" port: 46783 } attempt: 7
20:03:32.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:32.769927 22081 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.21.124.124" port: 45763 } has no permanent_uuid. Determining permanent_uuid...
20:03:32.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:32.795184 22081 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.21.124.124:45763: Network error: Client connection negotiation failed: client connection to 127.21.124.124:45763: connect: Connection refused (error 111)
20:03:32.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:32.853804 22081 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.124.124" port: 45763 } attempt: 1
20:03:32.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:32.863044 22081 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.21.124.124:45763: Network error: Client connection negotiation failed: client connection to 127.21.124.124:45763: connect: Connection refused (error 111)
20:03:32.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:32.944761 22081 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.124.124" port: 45763 } attempt: 2
20:03:32.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:32.952865 22081 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.21.124.124:45763: Network error: Client connection negotiation failed: client connection to 127.21.124.124:45763: connect: Connection refused (error 111)
20:03:33.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:33.038501 22081 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.124.124" port: 45763 } attempt: 3
20:03:33.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:33.040884 22156 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.124.124" port: 45763 } attempt: 5
20:03:33.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:33.055111 22081 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.21.124.124:45763: Network error: Client connection negotiation failed: client connection to 127.21.124.124:45763: connect: Connection refused (error 111)
20:03:33.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:33.058911 22156 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.21.124.124:45763: Network error: Client connection negotiation failed: client connection to 127.21.124.124:45763: connect: Connection refused (error 111)
20:03:33.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:33.195834 22081 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.124.124" port: 45763 } attempt: 4
20:03:33.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:33.205024 22081 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.21.124.124:45763: Network error: Client connection negotiation failed: client connection to 127.21.124.124:45763: connect: Connection refused (error 111)
20:03:33.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:33.488677 22081 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.124.124" port: 45763 } attempt: 5
20:03:33.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:33.497838 22081 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.21.124.124:45763: Network error: Client connection negotiation failed: client connection to 127.21.124.124:45763: connect: Connection refused (error 111)
20:03:33.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:33.606760 22156 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.124.124" port: 45763 } attempt: 6
20:03:33.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:33.633865 22156 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.21.124.124:45763: Network error: Client connection negotiation failed: client connection to 127.21.124.124:45763: connect: Connection refused (error 111)
20:03:34.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:34.050500 22081 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.124.124" port: 45763 } attempt: 6
20:03:34.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:34.075307 22081 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.21.124.124:45763: Network error: Client connection negotiation failed: client connection to 127.21.124.124:45763: connect: Connection refused (error 111)
20:03:34.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:34.127480 22167 debug-util.cc:398] Leaking SignalData structure 0x7b0800034cc0 after lost signal to thread 22158
20:03:34.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:34.415652 22158 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.693s user 0.584s sys 1.104s
20:03:34.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:34.417791 22158 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.695s user 0.584s sys 1.105s
20:03:34.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:32.724121 22169 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:34.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:34.420675 22171 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:34.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:34.438798 22170 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1714 milliseconds
20:03:34.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:34.438750 22158 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:03:34.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:34.440215 22158 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:03:34.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:34.443468 22158 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:03:34.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:34.445014 22158 hybrid_clock.cc:648] HybridClock initialized: now 1777838614444991 us; error 55 us; skew 500 ppm
20:03:34.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:34.446039 22158 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:03:34.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:34.452507 22158 webserver.cc:492] Webserver started at http://127.21.124.124:35799/ using document root <none> and password file <none>
20:03:34.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:34.453696 22158 fs_manager.cc:362] Metadata directory not provided
20:03:34.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:34.453964 22158 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:03:34.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:34.454636 22158 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
20:03:34.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:34.460284 22158 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster11212040801989529672/master-2/data/instance:
20:03:34.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f90ef18580774d77864909348b5b0693"
20:03:34.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-03 20:03:34 on dist-test-slave-b0rl"
20:03:34.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:34.461768 22158 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster11212040801989529672/master-2/wal/instance:
20:03:34.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f90ef18580774d77864909348b5b0693"
20:03:34.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-03 20:03:34 on dist-test-slave-b0rl"
20:03:34.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:34.470238 22158 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.005s sys 0.001s
20:03:34.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:34.476959 22185 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:03:34.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:34.478583 22158 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.004s sys 0.001s
20:03:34.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:34.479137 22158 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster11212040801989529672/master-2/data,/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster11212040801989529672/master-2/wal
20:03:34.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f90ef18580774d77864909348b5b0693"
20:03:34.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-03 20:03:34 on dist-test-slave-b0rl"
20:03:34.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:34.479486 22158 fs_report.cc:389] FS layout report
20:03:34.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:03:34.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster11212040801989529672/master-2/wal
20:03:34.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster11212040801989529672/master-2/wal
20:03:34.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster11212040801989529672/master-2/data/data
20:03:34.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:03:34.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:03:34.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:03:34.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:03:34.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:03:34.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:03:34.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:03:34.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:03:34.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:03:34.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:03:34.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:03:34.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:34.536350 22158 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:03:34.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:34.538235 22158 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:03:34.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:34.538781 22158 kserver.cc:163] Server-wide thread pool size limit: 3276
20:03:34.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:34.619289 22158 rpc_server.cc:307] RPC server started. Bound to: 127.21.124.124:45763
20:03:34.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:34.619379 22236 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.21.124.124:45763 every 8 connection(s)
20:03:34.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:34.621912 22158 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster11212040801989529672/master-2/data/info.pb
20:03:34.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:34.628186 22237 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
20:03:34.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:34.629021 22001 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskqmogBp/build/tsan/bin/kudu as pid 22158
20:03:34.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:34.629348 22001 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster11212040801989529672/master-2/wal/instance
20:03:34.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:34.646800 22237 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.21.124.126" port: 41563 } has no permanent_uuid. Determining permanent_uuid...
20:03:34.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:34.672801 22237 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.21.124.125" port: 46783 } has no permanent_uuid. Determining permanent_uuid...
20:03:34.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:34.686334 22237 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.21.124.124" port: 45763 } has no permanent_uuid. Determining permanent_uuid...
20:03:34.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:34.694459 22156 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.124.124" port: 45763 } attempt: 7
20:03:34.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:34.709187 22237 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f90ef18580774d77864909348b5b0693: Bootstrap starting.
20:03:34.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:34.715932 22237 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P f90ef18580774d77864909348b5b0693: Neither blocks nor log segments found. Creating new log.
20:03:34.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:34.717314 22156 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 8130d13960a14c88b4095c433ca69b07: Bootstrap starting.
20:03:34.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:34.718436 22237 log.cc:826] T 00000000000000000000000000000000 P f90ef18580774d77864909348b5b0693: Log is configured to *not* fsync() on all Append() calls
20:03:34.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:34.725922 22156 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 8130d13960a14c88b4095c433ca69b07: Neither blocks nor log segments found. Creating new log.
20:03:34.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:34.727473 22237 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f90ef18580774d77864909348b5b0693: No bootstrap required, opened a new log
20:03:34.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:34.728240 22156 log.cc:826] T 00000000000000000000000000000000 P 8130d13960a14c88b4095c433ca69b07: Log is configured to *not* fsync() on all Append() calls
20:03:34.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:34.733436 22156 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 8130d13960a14c88b4095c433ca69b07: No bootstrap required, opened a new log
20:03:34.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:34.751047 22237 raft_consensus.cc:359] T 00000000000000000000000000000000 P f90ef18580774d77864909348b5b0693 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c87e8c6d5a524a2a84f403153fd2287b" member_type: VOTER last_known_addr { host: "127.21.124.126" port: 41563 } } peers { permanent_uuid: "8130d13960a14c88b4095c433ca69b07" member_type: VOTER last_known_addr { host: "127.21.124.125" port: 46783 } } peers { permanent_uuid: "f90ef18580774d77864909348b5b0693" member_type: VOTER last_known_addr { host: "127.21.124.124" port: 45763 } }
20:03:34.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:34.751845 22237 raft_consensus.cc:385] T 00000000000000000000000000000000 P f90ef18580774d77864909348b5b0693 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:03:34.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:34.752100 22237 raft_consensus.cc:740] T 00000000000000000000000000000000 P f90ef18580774d77864909348b5b0693 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f90ef18580774d77864909348b5b0693, State: Initialized, Role: FOLLOWER
20:03:34.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:34.752876 22237 consensus_queue.cc:260] T 00000000000000000000000000000000 P f90ef18580774d77864909348b5b0693 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c87e8c6d5a524a2a84f403153fd2287b" member_type: VOTER last_known_addr { host: "127.21.124.126" port: 41563 } } peers { permanent_uuid: "8130d13960a14c88b4095c433ca69b07" member_type: VOTER last_known_addr { host: "127.21.124.125" port: 46783 } } peers { permanent_uuid: "f90ef18580774d77864909348b5b0693" member_type: VOTER last_known_addr { host: "127.21.124.124" port: 45763 } }
20:03:34.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:34.755904 22249 sys_catalog.cc:455] T 00000000000000000000000000000000 P f90ef18580774d77864909348b5b0693 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c87e8c6d5a524a2a84f403153fd2287b" member_type: VOTER last_known_addr { host: "127.21.124.126" port: 41563 } } peers { permanent_uuid: "8130d13960a14c88b4095c433ca69b07" member_type: VOTER last_known_addr { host: "127.21.124.125" port: 46783 } } peers { permanent_uuid: "f90ef18580774d77864909348b5b0693" member_type: VOTER last_known_addr { host: "127.21.124.124" port: 45763 } } }
20:03:34.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:34.757865 22237 sys_catalog.cc:565] T 00000000000000000000000000000000 P f90ef18580774d77864909348b5b0693 [sys.catalog]: configured and running, proceeding with master startup.
20:03:34.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:34.758571 22249 sys_catalog.cc:458] T 00000000000000000000000000000000 P f90ef18580774d77864909348b5b0693 [sys.catalog]: This master's current role is: FOLLOWER
20:03:34.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:34.761750 22156 raft_consensus.cc:359] T 00000000000000000000000000000000 P 8130d13960a14c88b4095c433ca69b07 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c87e8c6d5a524a2a84f403153fd2287b" member_type: VOTER last_known_addr { host: "127.21.124.126" port: 41563 } } peers { permanent_uuid: "8130d13960a14c88b4095c433ca69b07" member_type: VOTER last_known_addr { host: "127.21.124.125" port: 46783 } } peers { permanent_uuid: "f90ef18580774d77864909348b5b0693" member_type: VOTER last_known_addr { host: "127.21.124.124" port: 45763 } }
20:03:34.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:34.762813 22156 raft_consensus.cc:385] T 00000000000000000000000000000000 P 8130d13960a14c88b4095c433ca69b07 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:03:34.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:34.763370 22156 raft_consensus.cc:740] T 00000000000000000000000000000000 P 8130d13960a14c88b4095c433ca69b07 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8130d13960a14c88b4095c433ca69b07, State: Initialized, Role: FOLLOWER
20:03:34.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:34.764379 22156 consensus_queue.cc:260] T 00000000000000000000000000000000 P 8130d13960a14c88b4095c433ca69b07 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c87e8c6d5a524a2a84f403153fd2287b" member_type: VOTER last_known_addr { host: "127.21.124.126" port: 41563 } } peers { permanent_uuid: "8130d13960a14c88b4095c433ca69b07" member_type: VOTER last_known_addr { host: "127.21.124.125" port: 46783 } } peers { permanent_uuid: "f90ef18580774d77864909348b5b0693" member_type: VOTER last_known_addr { host: "127.21.124.124" port: 45763 } }
20:03:34.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:34.768194 22254 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8130d13960a14c88b4095c433ca69b07 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c87e8c6d5a524a2a84f403153fd2287b" member_type: VOTER last_known_addr { host: "127.21.124.126" port: 41563 } } peers { permanent_uuid: "8130d13960a14c88b4095c433ca69b07" member_type: VOTER last_known_addr { host: "127.21.124.125" port: 46783 } } peers { permanent_uuid: "f90ef18580774d77864909348b5b0693" member_type: VOTER last_known_addr { host: "127.21.124.124" port: 45763 } } }
20:03:34.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:34.769099 22254 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8130d13960a14c88b4095c433ca69b07 [sys.catalog]: This master's current role is: FOLLOWER
20:03:34.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:34.770654 22156 sys_catalog.cc:565] T 00000000000000000000000000000000 P 8130d13960a14c88b4095c433ca69b07 [sys.catalog]: configured and running, proceeding with master startup.
20:03:34.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:34.794390 22191 tablet.cc:2404] T 00000000000000000000000000000000 P f90ef18580774d77864909348b5b0693: Can't schedule compaction. Clean time has not been advanced past its initial value.
20:03:34.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:34.798655 22265 catalog_manager.cc:1568] T 00000000000000000000000000000000 P f90ef18580774d77864909348b5b0693: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:03:34.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:34.799420 22265 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:03:34.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:34.786942 22158 master_runner.cc:186] Error getting master registration for 127.21.124.126:41563: OK, instance_id { permanent_uuid: "c87e8c6d5a524a2a84f403153fd2287b" instance_seqno: 1777838610360741 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
20:03:34.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:34.832387 22158 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:03:34.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:34.842290 22273 catalog_manager.cc:1568] T 00000000000000000000000000000000 P 8130d13960a14c88b4095c433ca69b07: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:03:34.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:34.845500 22273 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:03:34.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:34.815251 22083 master_runner.cc:186] Error getting master registration for 127.21.124.126:41563: OK, instance_id { permanent_uuid: "c87e8c6d5a524a2a84f403153fd2287b" instance_seqno: 1777838610360741 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
20:03:34.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:34.858527 22083 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:03:34.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:34.979326 22110 tablet.cc:2404] T 00000000000000000000000000000000 P 8130d13960a14c88b4095c433ca69b07: Can't schedule compaction. Clean time has not been advanced past its initial value.
20:03:35.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:35.101706 22249 raft_consensus.cc:493] T 00000000000000000000000000000000 P f90ef18580774d77864909348b5b0693 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:03:35.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:35.102324 22249 raft_consensus.cc:515] T 00000000000000000000000000000000 P f90ef18580774d77864909348b5b0693 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c87e8c6d5a524a2a84f403153fd2287b" member_type: VOTER last_known_addr { host: "127.21.124.126" port: 41563 } } peers { permanent_uuid: "8130d13960a14c88b4095c433ca69b07" member_type: VOTER last_known_addr { host: "127.21.124.125" port: 46783 } } peers { permanent_uuid: "f90ef18580774d77864909348b5b0693" member_type: VOTER last_known_addr { host: "127.21.124.124" port: 45763 } }
20:03:35.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:35.108120 22131 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "f90ef18580774d77864909348b5b0693" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8130d13960a14c88b4095c433ca69b07" is_pre_election: true
20:03:35.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:35.108836 22081 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.124.124" port: 45763 } attempt: 7
20:03:35.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:35.108590 22249 leader_election.cc:290] T 00000000000000000000000000000000 P f90ef18580774d77864909348b5b0693 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers c87e8c6d5a524a2a84f403153fd2287b (127.21.124.126:41563), 8130d13960a14c88b4095c433ca69b07 (127.21.124.125:46783)
20:03:35.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:35.109064 22131 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 8130d13960a14c88b4095c433ca69b07 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate f90ef18580774d77864909348b5b0693 in term 0.
20:03:35.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:35.110538 22188 leader_election.cc:304] T 00000000000000000000000000000000 P f90ef18580774d77864909348b5b0693 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 8130d13960a14c88b4095c433ca69b07, f90ef18580774d77864909348b5b0693; no voters:
20:03:35.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:35.111557 22249 raft_consensus.cc:2804] T 00000000000000000000000000000000 P f90ef18580774d77864909348b5b0693 [term 0 FOLLOWER]: Leader pre-election won for term 1
20:03:35.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:35.112480 22249 raft_consensus.cc:493] T 00000000000000000000000000000000 P f90ef18580774d77864909348b5b0693 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
20:03:35.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:35.114173 22249 raft_consensus.cc:3060] T 00000000000000000000000000000000 P f90ef18580774d77864909348b5b0693 [term 0 FOLLOWER]: Advancing to term 1
20:03:35.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:35.136315 22249 raft_consensus.cc:515] T 00000000000000000000000000000000 P f90ef18580774d77864909348b5b0693 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c87e8c6d5a524a2a84f403153fd2287b" member_type: VOTER last_known_addr { host: "127.21.124.126" port: 41563 } } peers { permanent_uuid: "8130d13960a14c88b4095c433ca69b07" member_type: VOTER last_known_addr { host: "127.21.124.125" port: 46783 } } peers { permanent_uuid: "f90ef18580774d77864909348b5b0693" member_type: VOTER last_known_addr { host: "127.21.124.124" port: 45763 } }
20:03:35.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:35.138659 22081 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c87e8c6d5a524a2a84f403153fd2287b: Bootstrap starting.
20:03:35.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:35.147137 22131 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "f90ef18580774d77864909348b5b0693" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8130d13960a14c88b4095c433ca69b07"
20:03:35.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:35.146970 22249 leader_election.cc:290] T 00000000000000000000000000000000 P f90ef18580774d77864909348b5b0693 [CANDIDATE]: Term 1 election: Requested vote from peers c87e8c6d5a524a2a84f403153fd2287b (127.21.124.126:41563), 8130d13960a14c88b4095c433ca69b07 (127.21.124.125:46783)
20:03:35.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:35.147716 22131 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 8130d13960a14c88b4095c433ca69b07 [term 0 FOLLOWER]: Advancing to term 1
20:03:35.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:35.152542 22081 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P c87e8c6d5a524a2a84f403153fd2287b: Neither blocks nor log segments found. Creating new log.
20:03:35.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:35.154537 22131 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 8130d13960a14c88b4095c433ca69b07 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate f90ef18580774d77864909348b5b0693 in term 1.
20:03:35.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:35.156028 22188 leader_election.cc:304] T 00000000000000000000000000000000 P f90ef18580774d77864909348b5b0693 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 8130d13960a14c88b4095c433ca69b07, f90ef18580774d77864909348b5b0693; no voters:
20:03:35.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:35.159649 22249 raft_consensus.cc:2804] T 00000000000000000000000000000000 P f90ef18580774d77864909348b5b0693 [term 1 FOLLOWER]: Leader election won for term 1
20:03:35.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:35.162678 22081 log.cc:826] T 00000000000000000000000000000000 P c87e8c6d5a524a2a84f403153fd2287b: Log is configured to *not* fsync() on all Append() calls
20:03:35.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:35.174562 22249 raft_consensus.cc:697] T 00000000000000000000000000000000 P f90ef18580774d77864909348b5b0693 [term 1 LEADER]: Becoming Leader. State: Replica: f90ef18580774d77864909348b5b0693, State: Running, Role: LEADER
20:03:35.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:35.177224 22249 consensus_queue.cc:237] T 00000000000000000000000000000000 P f90ef18580774d77864909348b5b0693 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c87e8c6d5a524a2a84f403153fd2287b" member_type: VOTER last_known_addr { host: "127.21.124.126" port: 41563 } } peers { permanent_uuid: "8130d13960a14c88b4095c433ca69b07" member_type: VOTER last_known_addr { host: "127.21.124.125" port: 46783 } } peers { permanent_uuid: "f90ef18580774d77864909348b5b0693" member_type: VOTER last_known_addr { host: "127.21.124.124" port: 45763 } }
20:03:35.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:35.194787 22081 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c87e8c6d5a524a2a84f403153fd2287b: No bootstrap required, opened a new log
20:03:35.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:35.206005 22276 sys_catalog.cc:455] T 00000000000000000000000000000000 P f90ef18580774d77864909348b5b0693 [sys.catalog]: SysCatalogTable state changed. Reason: New leader f90ef18580774d77864909348b5b0693. Latest consensus state: current_term: 1 leader_uuid: "f90ef18580774d77864909348b5b0693" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c87e8c6d5a524a2a84f403153fd2287b" member_type: VOTER last_known_addr { host: "127.21.124.126" port: 41563 } } peers { permanent_uuid: "8130d13960a14c88b4095c433ca69b07" member_type: VOTER last_known_addr { host: "127.21.124.125" port: 46783 } } peers { permanent_uuid: "f90ef18580774d77864909348b5b0693" member_type: VOTER last_known_addr { host: "127.21.124.124" port: 45763 } } }
20:03:35.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:35.207484 22276 sys_catalog.cc:458] T 00000000000000000000000000000000 P f90ef18580774d77864909348b5b0693 [sys.catalog]: This master's current role is: LEADER
20:03:35.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:35.223768 22279 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
20:03:35.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:35.231894 22279 catalog_manager.cc:1494] Initializing Kudu cluster ID...
20:03:35.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:35.275666 22131 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 8130d13960a14c88b4095c433ca69b07 [term 1 FOLLOWER]: Refusing update from remote peer f90ef18580774d77864909348b5b0693: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
20:03:35.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:35.275558 22081 raft_consensus.cc:359] T 00000000000000000000000000000000 P c87e8c6d5a524a2a84f403153fd2287b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c87e8c6d5a524a2a84f403153fd2287b" member_type: VOTER last_known_addr { host: "127.21.124.126" port: 41563 } } peers { permanent_uuid: "8130d13960a14c88b4095c433ca69b07" member_type: VOTER last_known_addr { host: "127.21.124.125" port: 46783 } } peers { permanent_uuid: "f90ef18580774d77864909348b5b0693" member_type: VOTER last_known_addr { host: "127.21.124.124" port: 45763 } }
20:03:35.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:35.276423 22081 raft_consensus.cc:385] T 00000000000000000000000000000000 P c87e8c6d5a524a2a84f403153fd2287b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:03:35.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:35.276708 22081 raft_consensus.cc:740] T 00000000000000000000000000000000 P c87e8c6d5a524a2a84f403153fd2287b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c87e8c6d5a524a2a84f403153fd2287b, State: Initialized, Role: FOLLOWER
20:03:35.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:35.277594 22081 consensus_queue.cc:260] T 00000000000000000000000000000000 P c87e8c6d5a524a2a84f403153fd2287b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c87e8c6d5a524a2a84f403153fd2287b" member_type: VOTER last_known_addr { host: "127.21.124.126" port: 41563 } } peers { permanent_uuid: "8130d13960a14c88b4095c433ca69b07" member_type: VOTER last_known_addr { host: "127.21.124.125" port: 46783 } } peers { permanent_uuid: "f90ef18580774d77864909348b5b0693" member_type: VOTER last_known_addr { host: "127.21.124.124" port: 45763 } }
20:03:35.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:35.279336 22276 consensus_queue.cc:1048] T 00000000000000000000000000000000 P f90ef18580774d77864909348b5b0693 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8130d13960a14c88b4095c433ca69b07" member_type: VOTER last_known_addr { host: "127.21.124.125" port: 46783 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
20:03:35.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:35.290359 22081 sys_catalog.cc:565] T 00000000000000000000000000000000 P c87e8c6d5a524a2a84f403153fd2287b [sys.catalog]: configured and running, proceeding with master startup.
20:03:35.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:35.290530 22055 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "f90ef18580774d77864909348b5b0693" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c87e8c6d5a524a2a84f403153fd2287b"
20:03:35.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:35.291556 22056 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "f90ef18580774d77864909348b5b0693" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c87e8c6d5a524a2a84f403153fd2287b" is_pre_election: true
20:03:35.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:35.291273 22054 raft_consensus.cc:3060] T 00000000000000000000000000000000 P c87e8c6d5a524a2a84f403153fd2287b [term 0 FOLLOWER]: Advancing to term 1
20:03:35.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:35.318696 22054 raft_consensus.cc:1275] T 00000000000000000000000000000000 P c87e8c6d5a524a2a84f403153fd2287b [term 1 FOLLOWER]: Refusing update from remote peer f90ef18580774d77864909348b5b0693: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
20:03:35.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:35.319958 22035 tablet.cc:2404] T 00000000000000000000000000000000 P c87e8c6d5a524a2a84f403153fd2287b: Can't schedule compaction. Clean time has not been advanced past its initial value.
20:03:35.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:35.336755 22281 sys_catalog.cc:455] T 00000000000000000000000000000000 P c87e8c6d5a524a2a84f403153fd2287b [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c87e8c6d5a524a2a84f403153fd2287b" member_type: VOTER last_known_addr { host: "127.21.124.126" port: 41563 } } peers { permanent_uuid: "8130d13960a14c88b4095c433ca69b07" member_type: VOTER last_known_addr { host: "127.21.124.125" port: 46783 } } peers { permanent_uuid: "f90ef18580774d77864909348b5b0693" member_type: VOTER last_known_addr { host: "127.21.124.124" port: 45763 } } }
20:03:35.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:35.337826 22281 sys_catalog.cc:458] T 00000000000000000000000000000000 P c87e8c6d5a524a2a84f403153fd2287b [sys.catalog]: This master's current role is: FOLLOWER
20:03:35.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:35.338390 22276 consensus_queue.cc:1048] T 00000000000000000000000000000000 P f90ef18580774d77864909348b5b0693 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c87e8c6d5a524a2a84f403153fd2287b" member_type: VOTER last_known_addr { host: "127.21.124.126" port: 41563 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
20:03:35.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:35.365087 22282 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8130d13960a14c88b4095c433ca69b07 [sys.catalog]: SysCatalogTable state changed. Reason: New leader f90ef18580774d77864909348b5b0693. Latest consensus state: current_term: 1 leader_uuid: "f90ef18580774d77864909348b5b0693" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c87e8c6d5a524a2a84f403153fd2287b" member_type: VOTER last_known_addr { host: "127.21.124.126" port: 41563 } } peers { permanent_uuid: "8130d13960a14c88b4095c433ca69b07" member_type: VOTER last_known_addr { host: "127.21.124.125" port: 46783 } } peers { permanent_uuid: "f90ef18580774d77864909348b5b0693" member_type: VOTER last_known_addr { host: "127.21.124.124" port: 45763 } } }
20:03:35.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:35.366783 22282 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8130d13960a14c88b4095c433ca69b07 [sys.catalog]: This master's current role is: FOLLOWER
20:03:35.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:35.415642 22281 sys_catalog.cc:455] T 00000000000000000000000000000000 P c87e8c6d5a524a2a84f403153fd2287b [sys.catalog]: SysCatalogTable state changed. Reason: New leader f90ef18580774d77864909348b5b0693. Latest consensus state: current_term: 1 leader_uuid: "f90ef18580774d77864909348b5b0693" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c87e8c6d5a524a2a84f403153fd2287b" member_type: VOTER last_known_addr { host: "127.21.124.126" port: 41563 } } peers { permanent_uuid: "8130d13960a14c88b4095c433ca69b07" member_type: VOTER last_known_addr { host: "127.21.124.125" port: 46783 } } peers { permanent_uuid: "f90ef18580774d77864909348b5b0693" member_type: VOTER last_known_addr { host: "127.21.124.124" port: 45763 } } }
20:03:35.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:35.416404 22281 sys_catalog.cc:458] T 00000000000000000000000000000000 P c87e8c6d5a524a2a84f403153fd2287b [sys.catalog]: This master's current role is: FOLLOWER
20:03:35.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:35.416919 22280 mvcc.cc:204] Tried to move back new op lower bound from 7282026968111538176 to 7282026967788609536. Current Snapshot: MvccSnapshot[applied={T|T < 7282026968111538176}]
20:03:35.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:35.437474 22290 mvcc.cc:204] Tried to move back new op lower bound from 7282026968111538176 to 7282026967788609536. Current Snapshot: MvccSnapshot[applied={T|T < 7282026968111538176}]
20:03:35.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:35.443995 22276 sys_catalog.cc:455] T 00000000000000000000000000000000 P f90ef18580774d77864909348b5b0693 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "f90ef18580774d77864909348b5b0693" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c87e8c6d5a524a2a84f403153fd2287b" member_type: VOTER last_known_addr { host: "127.21.124.126" port: 41563 } } peers { permanent_uuid: "8130d13960a14c88b4095c433ca69b07" member_type: VOTER last_known_addr { host: "127.21.124.125" port: 46783 } } peers { permanent_uuid: "f90ef18580774d77864909348b5b0693" member_type: VOTER last_known_addr { host: "127.21.124.124" port: 45763 } } }
20:03:35.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:35.444777 22276 sys_catalog.cc:458] T 00000000000000000000000000000000 P f90ef18580774d77864909348b5b0693 [sys.catalog]: This master's current role is: LEADER
20:03:35.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:35.446913 22276 sys_catalog.cc:455] T 00000000000000000000000000000000 P f90ef18580774d77864909348b5b0693 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "f90ef18580774d77864909348b5b0693" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c87e8c6d5a524a2a84f403153fd2287b" member_type: VOTER last_known_addr { host: "127.21.124.126" port: 41563 } } peers { permanent_uuid: "8130d13960a14c88b4095c433ca69b07" member_type: VOTER last_known_addr { host: "127.21.124.125" port: 46783 } } peers { permanent_uuid: "f90ef18580774d77864909348b5b0693" member_type: VOTER last_known_addr { host: "127.21.124.124" port: 45763 } } }
20:03:35.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:35.447587 22276 sys_catalog.cc:458] T 00000000000000000000000000000000 P f90ef18580774d77864909348b5b0693 [sys.catalog]: This master's current role is: LEADER
20:03:35.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:35.449298 22279 catalog_manager.cc:1357] Generated new cluster ID: 13587676545d4c8a8860ba084513d7bb
20:03:35.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:35.449676 22279 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
20:03:35.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:35.470762 22281 sys_catalog.cc:455] T 00000000000000000000000000000000 P c87e8c6d5a524a2a84f403153fd2287b [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "f90ef18580774d77864909348b5b0693" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c87e8c6d5a524a2a84f403153fd2287b" member_type: VOTER last_known_addr { host: "127.21.124.126" port: 41563 } } peers { permanent_uuid: "8130d13960a14c88b4095c433ca69b07" member_type: VOTER last_known_addr { host: "127.21.124.125" port: 46783 } } peers { permanent_uuid: "f90ef18580774d77864909348b5b0693" member_type: VOTER last_known_addr { host: "127.21.124.124" port: 45763 } } }
20:03:35.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:35.471499 22281 sys_catalog.cc:458] T 00000000000000000000000000000000 P c87e8c6d5a524a2a84f403153fd2287b [sys.catalog]: This master's current role is: FOLLOWER
20:03:35.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:35.480659 22282 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8130d13960a14c88b4095c433ca69b07 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "f90ef18580774d77864909348b5b0693" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c87e8c6d5a524a2a84f403153fd2287b" member_type: VOTER last_known_addr { host: "127.21.124.126" port: 41563 } } peers { permanent_uuid: "8130d13960a14c88b4095c433ca69b07" member_type: VOTER last_known_addr { host: "127.21.124.125" port: 46783 } } peers { permanent_uuid: "f90ef18580774d77864909348b5b0693" member_type: VOTER last_known_addr { host: "127.21.124.124" port: 45763 } } }
20:03:35.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:35.481330 22282 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8130d13960a14c88b4095c433ca69b07 [sys.catalog]: This master's current role is: FOLLOWER
20:03:35.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:35.530954 22300 catalog_manager.cc:1269] Loaded cluster ID: 13587676545d4c8a8860ba084513d7bb
20:03:35.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:35.531246 22300 catalog_manager.cc:1562] T 00000000000000000000000000000000 P c87e8c6d5a524a2a84f403153fd2287b: loading cluster ID for follower catalog manager: success
20:03:35.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:35.547174 22300 catalog_manager.cc:1586] T 00000000000000000000000000000000 P c87e8c6d5a524a2a84f403153fd2287b: acquiring CA information for follower catalog manager: Not found: root CA entry not found
20:03:35.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:35.547523 22300 catalog_manager.cc:883] Not found: root CA entry not found: failed to prepare follower catalog manager, will retry
20:03:35.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:35.558471 22279 catalog_manager.cc:1380] Generated new certificate authority record
20:03:35.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:35.561133 22279 catalog_manager.cc:1514] Loading token signing keys...
20:03:35.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:35.630652 22279 catalog_manager.cc:6044] T 00000000000000000000000000000000 P f90ef18580774d77864909348b5b0693: Generated new TSK 0
20:03:35.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:35.632337 22279 catalog_manager.cc:1524] Initializing in-progress tserver states...
20:03:35.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:35.677964 22001 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskqmogBp/build/tsan/bin/kudu
20:03:35.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskqmogBp/build/tsan/bin/kudu
20:03:35.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster11212040801989529672/ts-0/wal
20:03:35.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster11212040801989529672/ts-0/data
20:03:35.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:03:35.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:03:35.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:03:35.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:03:35.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:03:35.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:03:35.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster11212040801989529672/ts-0/logs
20:03:35.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster11212040801989529672/ts-0/data/info.pb
20:03:35.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:03:35.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:03:35.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:03:35.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:03:35.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:03:35.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:03:35.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:03:35.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:03:35.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
20:03:35.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:03:35.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.124.65:0
20:03:35.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.21.124.65
20:03:35.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.124.65
20:03:35.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:03:35.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.21.124.126:41563,127.21.124.125:46783,127.21.124.124:45763
20:03:35.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.124.84:42337
20:03:35.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:03:35.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:03:35.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
20:03:35.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:35.852803 22273 catalog_manager.cc:1269] Loaded cluster ID: 13587676545d4c8a8860ba084513d7bb
20:03:35.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:35.853718 22273 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 8130d13960a14c88b4095c433ca69b07: loading cluster ID for follower catalog manager: success
20:03:35.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:35.872804 22273 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 8130d13960a14c88b4095c433ca69b07: acquiring CA information for follower catalog manager: success
20:03:35.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:35.880268 22273 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 8130d13960a14c88b4095c433ca69b07: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
20:03:36.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:36.281642 22302 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:03:36.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:36.282840 22302 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:03:36.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:36.283842 22302 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:03:36.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:36.346678 22302 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:03:36.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:36.348428 22302 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.21.124.65
20:03:36.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:36.417857 22302 tablet_server_runner.cc:78] Tablet server non-default flags:
20:03:36.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:03:36.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.124.84:42337
20:03:36.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:03:36.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:03:36.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster11212040801989529672/ts-0/data
20:03:36.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster11212040801989529672/ts-0/wal
20:03:36.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:03:36.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:03:36.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.124.65:0
20:03:36.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:03:36.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:03:36.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:03:36.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster11212040801989529672/ts-0/data/info.pb
20:03:36.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.124.65
20:03:36.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:03:36.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.21.124.126:41563,127.21.124.125:46783,127.21.124.124:45763
20:03:36.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:03:36.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:03:36.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:03:36.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:03:36.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:03:36.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.21.124.65
20:03:36.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster11212040801989529672/ts-0/logs
20:03:36.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:03:36.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:03:36.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:03:36.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
20:03:36.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:03:36.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0ba7f54e15c9f673bf3e350f30b8d5f1a4037b79
20:03:36.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:03:36.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 03 May 2026 19:43:25 UTC on e7f111948823
20:03:36.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11731
20:03:36.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:03:36.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:36.428493 22302 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:03:36.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:36.431478 22302 file_cache.cc:492] Constructed file cache file cache with capacity 419430
20:03:36.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:36.497169 22309 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:36.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:36.584606 22300 catalog_manager.cc:1584] T 00000000000000000000000000000000 P c87e8c6d5a524a2a84f403153fd2287b: acquiring CA information for follower catalog manager: success
20:03:36.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:36.619607 22300 catalog_manager.cc:1612] T 00000000000000000000000000000000 P c87e8c6d5a524a2a84f403153fd2287b: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
20:03:37.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:37.545055 22152 debug-util.cc:398] Leaking SignalData structure 0x7b08000ade00 after lost signal to thread 22087
20:03:37.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:37.546094 22152 debug-util.cc:398] Leaking SignalData structure 0x7b08000add20 after lost signal to thread 22155
20:03:39.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:37.869797 22308 debug-util.cc:398] Leaking SignalData structure 0x7b0800034ca0 after lost signal to thread 22302
20:03:39.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:39.124832 22308 kernel_stack_watchdog.cc:198] Thread 22302 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 366ms:
20:03:39.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
20:03:39.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
20:03:39.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
20:03:39.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
20:03:39.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:39.127990 22312 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:39.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:39.132421 22302 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 2.631s user 0.618s sys 0.908s
20:03:39.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:39.135279 22302 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 2.634s user 0.619s sys 0.908s
20:03:39.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:39.135824 22302 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:03:39.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:39.134951 22311 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection timed out after 2632 milliseconds
20:03:39.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:36.503762 22310 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:39.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:39.138572 22302 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:03:39.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:39.147266 22302 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:03:39.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:39.149153 22302 hybrid_clock.cc:648] HybridClock initialized: now 1777838619149101 us; error 72 us; skew 500 ppm
20:03:39.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:39.150688 22302 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:03:39.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:39.170221 22302 webserver.cc:492] Webserver started at http://127.21.124.65:40985/ using document root <none> and password file <none>
20:03:39.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:39.171993 22302 fs_manager.cc:362] Metadata directory not provided
20:03:39.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:39.172555 22302 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:03:39.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:39.173336 22302 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
20:03:39.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:39.180089 22302 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster11212040801989529672/ts-0/data/instance:
20:03:39.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9f06c53d673946c3bae12555e56cfc13"
20:03:39.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-03 20:03:39 on dist-test-slave-b0rl"
20:03:39.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:39.182065 22302 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster11212040801989529672/ts-0/wal/instance:
20:03:39.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9f06c53d673946c3bae12555e56cfc13"
20:03:39.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-03 20:03:39 on dist-test-slave-b0rl"
20:03:39.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:39.198040 22302 fs_manager.cc:696] Time spent creating directory manager: real 0.015s user 0.004s sys 0.008s
20:03:39.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:39.212708 22322 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:03:39.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:39.214615 22302 fs_manager.cc:730] Time spent opening block manager: real 0.011s user 0.008s sys 0.001s
20:03:39.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:39.215387 22302 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster11212040801989529672/ts-0/data,/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster11212040801989529672/ts-0/wal
20:03:39.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9f06c53d673946c3bae12555e56cfc13"
20:03:39.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-03 20:03:39 on dist-test-slave-b0rl"
20:03:39.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:39.216244 22302 fs_report.cc:389] FS layout report
20:03:39.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:03:39.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster11212040801989529672/ts-0/wal
20:03:39.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster11212040801989529672/ts-0/wal
20:03:39.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster11212040801989529672/ts-0/data/data
20:03:39.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:03:39.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:03:39.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:03:39.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:03:39.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:03:39.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:03:39.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:03:39.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:03:39.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:03:39.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:03:39.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:03:39.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:39.311234 22302 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:03:39.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:39.313304 22302 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:03:39.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:39.314374 22302 kserver.cc:163] Server-wide thread pool size limit: 3276
20:03:39.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:39.318152 22302 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
20:03:39.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:39.324906 22302 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
20:03:39.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:39.325234 22302 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
20:03:39.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:39.325552 22302 ts_tablet_manager.cc:616] Registered 0 tablets
20:03:39.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:39.325784 22302 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
20:03:39.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:39.718752 22302 rpc_server.cc:307] RPC server started. Bound to: 127.21.124.65:37853
20:03:39.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:39.718937 22434 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.21.124.65:37853 every 8 connection(s)
20:03:39.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:39.722339 22302 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster11212040801989529672/ts-0/data/info.pb
20:03:39.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:39.732247 22001 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskqmogBp/build/tsan/bin/kudu as pid 22302
20:03:39.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:39.733073 22001 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster11212040801989529672/ts-0/wal/instance
20:03:39.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:39.766718 22001 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskqmogBp/build/tsan/bin/kudu
20:03:39.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskqmogBp/build/tsan/bin/kudu
20:03:39.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster11212040801989529672/ts-1/wal
20:03:39.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster11212040801989529672/ts-1/data
20:03:39.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:03:39.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:03:39.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:03:39.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:03:39.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:03:39.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:03:39.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster11212040801989529672/ts-1/logs
20:03:39.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster11212040801989529672/ts-1/data/info.pb
20:03:39.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:03:39.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:03:39.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:03:39.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:03:39.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:03:39.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:03:39.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:03:39.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:03:39.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
20:03:39.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:03:39.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.124.66:0
20:03:39.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.21.124.66
20:03:39.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.124.66
20:03:39.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:03:39.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.21.124.126:41563,127.21.124.125:46783,127.21.124.124:45763
20:03:39.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.124.84:42337
20:03:39.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:03:39.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:03:39.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
20:03:39.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:39.798697 22436 heartbeater.cc:344] Connected to a master server at 127.21.124.125:46783
20:03:39.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:39.799270 22436 heartbeater.cc:461] Registering TS with master...
20:03:39.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:39.800714 22436 heartbeater.cc:507] Master 127.21.124.125:46783 requested a full tablet report, sending...
20:03:39.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:39.809561 22121 ts_manager.cc:194] Registered new tserver with Master: 9f06c53d673946c3bae12555e56cfc13 (127.21.124.65:37853)
20:03:39.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:39.807385 22435 heartbeater.cc:344] Connected to a master server at 127.21.124.124:45763
20:03:39.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:39.816152 22435 heartbeater.cc:461] Registering TS with master...
20:03:39.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:39.817133 22435 heartbeater.cc:507] Master 127.21.124.124:45763 requested a full tablet report, sending...
20:03:39.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:39.820055 22202 ts_manager.cc:194] Registered new tserver with Master: 9f06c53d673946c3bae12555e56cfc13 (127.21.124.65:37853)
20:03:39.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:39.822726 22202 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.21.124.65:58313
20:03:39.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:39.839630 22439 heartbeater.cc:344] Connected to a master server at 127.21.124.126:41563
20:03:39.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:39.840036 22439 heartbeater.cc:461] Registering TS with master...
20:03:39.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:39.840698 22439 heartbeater.cc:507] Master 127.21.124.126:41563 requested a full tablet report, sending...
20:03:39.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:39.844834 22046 ts_manager.cc:194] Registered new tserver with Master: 9f06c53d673946c3bae12555e56cfc13 (127.21.124.65:37853)
20:03:40.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:40.308580 22442 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:03:40.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:40.309232 22442 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:03:40.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:40.309939 22442 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:03:40.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:40.379998 22442 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:03:40.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:40.381999 22442 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.21.124.66
20:03:40.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:40.467007 22442 tablet_server_runner.cc:78] Tablet server non-default flags:
20:03:40.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:03:40.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.124.84:42337
20:03:40.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:03:40.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:03:40.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster11212040801989529672/ts-1/data
20:03:40.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster11212040801989529672/ts-1/wal
20:03:40.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:03:40.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:03:40.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.124.66:0
20:03:40.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:03:40.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:03:40.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:03:40.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster11212040801989529672/ts-1/data/info.pb
20:03:40.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.124.66
20:03:40.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:03:40.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.21.124.126:41563,127.21.124.125:46783,127.21.124.124:45763
20:03:40.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:03:40.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:03:40.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:03:40.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:03:40.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:03:40.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.21.124.66
20:03:40.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster11212040801989529672/ts-1/logs
20:03:40.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:03:40.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:03:40.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:03:40.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
20:03:40.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:03:40.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0ba7f54e15c9f673bf3e350f30b8d5f1a4037b79
20:03:40.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:03:40.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 03 May 2026 19:43:25 UTC on e7f111948823
20:03:40.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11731
20:03:40.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:03:40.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:40.468731 22442 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:03:40.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:40.471329 22442 file_cache.cc:492] Constructed file cache file cache with capacity 419430
20:03:40.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:40.504110 22451 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:40.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:40.827368 22435 heartbeater.cc:499] Master 127.21.124.124:45763 was elected leader, sending a full tablet report...
20:03:42.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:41.907827 22450 debug-util.cc:398] Leaking SignalData structure 0x7b0800034ca0 after lost signal to thread 22442
20:03:42.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:42.011057 22450 kernel_stack_watchdog.cc:198] Thread 22442 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 399ms:
20:03:42.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
20:03:42.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
20:03:42.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
20:03:42.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
20:03:42.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:40.506618 22452 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:42.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:42.012821 22442 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.507s user 0.000s sys 0.003s
20:03:42.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:42.013221 22442 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.507s user 0.000s sys 0.003s
20:03:42.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:42.015686 22454 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:42.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:42.020004 22453 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1488 milliseconds
20:03:42.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:42.020017 22442 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:03:42.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:42.021363 22442 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:03:42.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:42.023550 22442 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:03:42.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:42.025004 22442 hybrid_clock.cc:648] HybridClock initialized: now 1777838622024998 us; error 73 us; skew 500 ppm
20:03:42.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:42.025986 22442 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:03:42.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:42.032538 22442 webserver.cc:492] Webserver started at http://127.21.124.66:34943/ using document root <none> and password file <none>
20:03:42.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:42.033589 22442 fs_manager.cc:362] Metadata directory not provided
20:03:42.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:42.033844 22442 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:03:42.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:42.034396 22442 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
20:03:42.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:42.038834 22442 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster11212040801989529672/ts-1/data/instance:
20:03:42.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "35398d85d0f043ed900a420c26408738"
20:03:42.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-03 20:03:42 on dist-test-slave-b0rl"
20:03:42.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:42.040095 22442 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster11212040801989529672/ts-1/wal/instance:
20:03:42.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "35398d85d0f043ed900a420c26408738"
20:03:42.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-03 20:03:42 on dist-test-slave-b0rl"
20:03:42.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:42.047796 22442 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.004s sys 0.004s
20:03:42.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:42.053673 22462 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:03:42.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:42.054932 22442 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.001s
20:03:42.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:42.055330 22442 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster11212040801989529672/ts-1/data,/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster11212040801989529672/ts-1/wal
20:03:42.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "35398d85d0f043ed900a420c26408738"
20:03:42.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-03 20:03:42 on dist-test-slave-b0rl"
20:03:42.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:42.055716 22442 fs_report.cc:389] FS layout report
20:03:42.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:03:42.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster11212040801989529672/ts-1/wal
20:03:42.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster11212040801989529672/ts-1/wal
20:03:42.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster11212040801989529672/ts-1/data/data
20:03:42.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:03:42.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:03:42.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:03:42.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:03:42.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:03:42.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:03:42.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:03:42.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:03:42.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:03:42.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:03:42.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:03:42.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:42.122104 22442 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:03:42.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:42.123793 22442 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:03:42.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:42.124352 22442 kserver.cc:163] Server-wide thread pool size limit: 3276
20:03:42.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:42.127113 22442 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
20:03:42.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:42.131527 22442 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
20:03:42.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:42.131786 22442 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
20:03:42.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:42.132092 22442 ts_tablet_manager.cc:616] Registered 0 tablets
20:03:42.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:42.132311 22442 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
20:03:42.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:42.296887 22442 rpc_server.cc:307] RPC server started. Bound to: 127.21.124.66:34317
20:03:42.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:42.297418 22574 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.21.124.66:34317 every 8 connection(s)
20:03:42.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:42.299888 22442 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster11212040801989529672/ts-1/data/info.pb
20:03:42.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:42.309065 22001 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskqmogBp/build/tsan/bin/kudu as pid 22442
20:03:42.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:42.309912 22001 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster11212040801989529672/ts-1/wal/instance
20:03:42.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:42.335053 22001 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskqmogBp/build/tsan/bin/kudu
20:03:42.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskqmogBp/build/tsan/bin/kudu
20:03:42.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster11212040801989529672/ts-2/wal
20:03:42.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster11212040801989529672/ts-2/data
20:03:42.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:03:42.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:03:42.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:03:42.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:03:42.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:03:42.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:03:42.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster11212040801989529672/ts-2/logs
20:03:42.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster11212040801989529672/ts-2/data/info.pb
20:03:42.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:03:42.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:03:42.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:03:42.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:03:42.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:03:42.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:03:42.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:03:42.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:03:42.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
20:03:42.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:03:42.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.124.67:0
20:03:42.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.21.124.67
20:03:42.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.124.67
20:03:42.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:03:42.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.21.124.126:41563,127.21.124.125:46783,127.21.124.124:45763
20:03:42.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.124.84:42337
20:03:42.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:03:42.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:03:42.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
20:03:42.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:42.354382 22575 heartbeater.cc:344] Connected to a master server at 127.21.124.124:45763
20:03:42.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:42.354983 22575 heartbeater.cc:461] Registering TS with master...
20:03:42.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:42.356858 22575 heartbeater.cc:507] Master 127.21.124.124:45763 requested a full tablet report, sending...
20:03:42.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:42.360265 22202 ts_manager.cc:194] Registered new tserver with Master: 35398d85d0f043ed900a420c26408738 (127.21.124.66:34317)
20:03:42.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:42.362126 22202 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.21.124.66:55393
20:03:42.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:42.365881 22576 heartbeater.cc:344] Connected to a master server at 127.21.124.125:46783
20:03:42.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:42.366266 22576 heartbeater.cc:461] Registering TS with master...
20:03:42.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:42.367086 22576 heartbeater.cc:507] Master 127.21.124.125:46783 requested a full tablet report, sending...
20:03:42.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:42.370687 22121 ts_manager.cc:194] Registered new tserver with Master: 35398d85d0f043ed900a420c26408738 (127.21.124.66:34317)
20:03:42.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:42.372027 22577 heartbeater.cc:344] Connected to a master server at 127.21.124.126:41563
20:03:42.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:42.372412 22577 heartbeater.cc:461] Registering TS with master...
20:03:42.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:42.373065 22577 heartbeater.cc:507] Master 127.21.124.126:41563 requested a full tablet report, sending...
20:03:42.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:42.375661 22046 ts_manager.cc:194] Registered new tserver with Master: 35398d85d0f043ed900a420c26408738 (127.21.124.66:34317)
20:03:42.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:42.688947 22586 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:03:42.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:42.689435 22586 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:03:42.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:42.689854 22586 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:03:42.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:42.723415 22586 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:03:42.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:42.724196 22586 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.21.124.67
20:03:42.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:42.761221 22586 tablet_server_runner.cc:78] Tablet server non-default flags:
20:03:42.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:03:42.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.124.84:42337
20:03:42.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:03:42.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:03:42.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster11212040801989529672/ts-2/data
20:03:42.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster11212040801989529672/ts-2/wal
20:03:42.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:03:42.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:03:42.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.124.67:0
20:03:42.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:03:42.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:03:42.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:03:42.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster11212040801989529672/ts-2/data/info.pb
20:03:42.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.124.67
20:03:42.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:03:42.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.21.124.126:41563,127.21.124.125:46783,127.21.124.124:45763
20:03:42.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:03:42.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:03:42.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:03:42.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:03:42.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:03:42.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.21.124.67
20:03:42.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster11212040801989529672/ts-2/logs
20:03:42.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:03:42.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:03:42.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:03:42.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
20:03:42.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:03:42.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0ba7f54e15c9f673bf3e350f30b8d5f1a4037b79
20:03:42.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:03:42.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 03 May 2026 19:43:25 UTC on e7f111948823
20:03:42.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11731
20:03:42.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:03:42.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:42.762493 22586 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:03:42.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:42.764087 22586 file_cache.cc:492] Constructed file cache file cache with capacity 419430
20:03:42.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:42.778504 22594 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:43.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:43.366916 22575 heartbeater.cc:499] Master 127.21.124.124:45763 was elected leader, sending a full tablet report...
20:03:44.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:42.778877 22595 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:44.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:44.126729 22597 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:44.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:44.129694 22586 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.352s user 0.433s sys 0.896s
20:03:44.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:44.130067 22586 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.352s user 0.433s sys 0.896s
20:03:44.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:44.131633 22596 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1344 milliseconds
20:03:44.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:44.131633 22586 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:03:44.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:44.133086 22586 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:03:44.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:44.135381 22586 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:03:44.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:44.136945 22586 hybrid_clock.cc:648] HybridClock initialized: now 1777838624136876 us; error 89 us; skew 500 ppm
20:03:44.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:44.137861 22586 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:03:44.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:44.147004 22586 webserver.cc:492] Webserver started at http://127.21.124.67:43619/ using document root <none> and password file <none>
20:03:44.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:44.148134 22586 fs_manager.cc:362] Metadata directory not provided
20:03:44.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:44.148428 22586 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:03:44.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:44.149000 22586 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
20:03:44.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:44.153519 22586 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster11212040801989529672/ts-2/data/instance:
20:03:44.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3ffd23f9ded543f89bf6f22611bd2d76"
20:03:44.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-03 20:03:44 on dist-test-slave-b0rl"
20:03:44.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:44.154917 22586 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster11212040801989529672/ts-2/wal/instance:
20:03:44.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3ffd23f9ded543f89bf6f22611bd2d76"
20:03:44.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-03 20:03:44 on dist-test-slave-b0rl"
20:03:44.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:44.164353 22586 fs_manager.cc:696] Time spent creating directory manager: real 0.009s user 0.006s sys 0.002s
20:03:44.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:44.171607 22604 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:03:44.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:44.173413 22586 fs_manager.cc:730] Time spent opening block manager: real 0.006s user 0.001s sys 0.003s
20:03:44.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:44.173852 22586 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster11212040801989529672/ts-2/data,/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster11212040801989529672/ts-2/wal
20:03:44.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3ffd23f9ded543f89bf6f22611bd2d76"
20:03:44.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-03 20:03:44 on dist-test-slave-b0rl"
20:03:44.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:44.174319 22586 fs_report.cc:389] FS layout report
20:03:44.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:03:44.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster11212040801989529672/ts-2/wal
20:03:44.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster11212040801989529672/ts-2/wal
20:03:44.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster11212040801989529672/ts-2/data/data
20:03:44.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:03:44.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:03:44.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:03:44.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:03:44.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:03:44.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:03:44.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:03:44.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:03:44.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:03:44.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:03:44.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:03:44.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:44.290190 22586 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:03:44.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:44.292634 22586 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:03:44.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:44.293190 22586 kserver.cc:163] Server-wide thread pool size limit: 3276
20:03:44.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:44.297749 22586 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
20:03:44.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:44.303452 22586 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
20:03:44.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:44.303723 22586 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
20:03:44.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:44.304035 22586 ts_tablet_manager.cc:616] Registered 0 tablets
20:03:44.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:44.304256 22586 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
20:03:44.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:44.479460 22586 rpc_server.cc:307] RPC server started. Bound to: 127.21.124.67:43165
20:03:44.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:44.479558 22716 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.21.124.67:43165 every 8 connection(s)
20:03:44.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:44.482079 22586 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster11212040801989529672/ts-2/data/info.pb
20:03:44.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:44.493983 22001 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskqmogBp/build/tsan/bin/kudu as pid 22586
20:03:44.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:44.494542 22001 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster11212040801989529672/ts-2/wal/instance
20:03:44.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:44.534421 22717 heartbeater.cc:344] Connected to a master server at 127.21.124.124:45763
20:03:44.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:44.535036 22717 heartbeater.cc:461] Registering TS with master...
20:03:44.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:44.536527 22717 heartbeater.cc:507] Master 127.21.124.124:45763 requested a full tablet report, sending...
20:03:44.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:44.539800 22202 ts_manager.cc:194] Registered new tserver with Master: 3ffd23f9ded543f89bf6f22611bd2d76 (127.21.124.67:43165)
20:03:44.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:44.540936 22719 heartbeater.cc:344] Connected to a master server at 127.21.124.126:41563
20:03:44.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:44.541296 22719 heartbeater.cc:461] Registering TS with master...
20:03:44.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:44.541286 22718 heartbeater.cc:344] Connected to a master server at 127.21.124.125:46783
20:03:44.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:44.541723 22718 heartbeater.cc:461] Registering TS with master...
20:03:44.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:44.541725 22202 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.21.124.67:43151
20:03:44.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:44.542276 22719 heartbeater.cc:507] Master 127.21.124.126:41563 requested a full tablet report, sending...
20:03:44.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:44.542666 22718 heartbeater.cc:507] Master 127.21.124.125:46783 requested a full tablet report, sending...
20:03:44.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:44.545959 22046 ts_manager.cc:194] Registered new tserver with Master: 3ffd23f9ded543f89bf6f22611bd2d76 (127.21.124.67:43165)
20:03:44.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:44.549996 22121 ts_manager.cc:194] Registered new tserver with Master: 3ffd23f9ded543f89bf6f22611bd2d76 (127.21.124.67:43165)
20:03:44.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:44.555433 22001 external_mini_cluster.cc:949] 3 TS(s) registered with all masters
20:03:44.569 [DEBUG - main] (MiniKuduCluster.java:195) Response:
20:03:44.570 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
20:03:44.579 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.21.124.126"
port: 41563
}
bound_http_address {
host: "127.21.124.126"
port: 34911
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.21.124.125"
port: 46783
}
bound_http_address {
host: "127.21.124.125"
port: 38195
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.21.124.124"
port: 45763
}
bound_http_address {
host: "127.21.124.124"
port: 35799
}
}
}
20:03:44.583 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
20:03:44.586 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.21.124.65"
port: 37853
}
bound_http_address {
host: "127.21.124.65"
port: 40985
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.21.124.66"
port: 34317
}
bound_http_address {
host: "127.21.124.66"
port: 34943
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.21.124.67"
port: 43165
}
bound_http_address {
host: "127.21.124.67"
port: 43619
}
}
}
20:03:44.586 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
20:03:44.798 [DEBUG - main] (CompositeStrategy.java:59) Unable to load modern Subject methods, relying only on legacy methods
java.lang.NoSuchMethodException: javax.security.auth.Subject.current()
at java.base/java.lang.Class.getDeclaredMethod(Class.java:2677)
at org.apache.kudu.client.internals.ModernStrategy.<init>(ModernStrategy.java:51)
at org.apache.kudu.client.internals.CompositeStrategy.<init>(CompositeStrategy.java:52)
at org.apache.kudu.client.internals.CompositeStrategy.<clinit>(CompositeStrategy.java:41)
at org.apache.kudu.client.internals.SecurityManagerCompatibility.get(SecurityManagerCompatibility.java:43)
at org.apache.kudu.client.SecurityContext.setupSubject(SecurityContext.java:155)
at org.apache.kudu.client.SecurityContext.<init>(SecurityContext.java:138)
at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:431)
at org.apache.kudu.client.AsyncKuduClient$AsyncKuduClientBuilder.build(AsyncKuduClient.java:3109)
at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:161)
at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runners.Suite.runChild(Suite.java:128)
at org.junit.runners.Suite.runChild(Suite.java:27)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runner.JUnitCore.run(JUnitCore.java:137)
at org.junit.runner.JUnitCore.run(JUnitCore.java:115)
at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77)
at org.junit.runner.JUnitCore.main(JUnitCore.java:36)
java.lang.NoSuchMethodException: javax.security.auth.Subject.current()
at java.base/java.lang.Class.getDeclaredMethod(Class.java:2677)
at org.apache.kudu.client.internals.ModernStrategy.<init>(ModernStrategy.java:51)
at org.apache.kudu.client.internals.CompositeStrategy.<init>(CompositeStrategy.java:52)
at org.apache.kudu.client.internals.CompositeStrategy.<clinit>(CompositeStrategy.java:41)
at org.apache.kudu.client.internals.SecurityManagerCompatibility.get(SecurityManagerCompatibility.java:43)
at org.apache.kudu.client.SecurityContext.setupSubject(SecurityContext.java:155)
at org.apache.kudu.client.SecurityContext.<init>(SecurityContext.java:138)
at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:431)
at org.apache.kudu.client.AsyncKuduClient$AsyncKuduClientBuilder.build(AsyncKuduClient.java:3109)
at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:161)
at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runners.Suite.runChild(Suite.java:128)
at org.junit.runners.Suite.runChild(Suite.java:27)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runner.JUnitCore.run(JUnitCore.java:137)
at org.junit.runner.JUnitCore.run(JUnitCore.java:115)
at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77)
at org.junit.runner.JUnitCore.main(JUnitCore.java:36)
20:03:44.858 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
20:03:44.961 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskqmogBp/test-tmp
20:03:44.962 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
20:03:44.962 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskqmogBp/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
20:03:44.984 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132"
extra_master_flags: "--webserver_require_spnego=false"
extra_master_flags: "--authn_token_validity_seconds=10"
extra_tserver_flags: "--webserver_require_spnego=false"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
20:03:45.346 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:03:45.346 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
20:03:45.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:45.349715 22729 mini_oidc.cc:150] Starting JWKS server
20:03:45.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:45.353555 22729 webserver.cc:492] Webserver started at http://127.0.0.1:32783/ using document root <none> and password file <none>
20:03:45.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:45.353965 22729 mini_oidc.cc:186] Starting OIDC Discovery server
20:03:45.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:45.355971 22729 webserver.cc:492] Webserver started at http://127.0.0.1:42131/ using document root <none> and password file <none>
20:03:45.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
20:03:45.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/krb5kdc/principal' for realm 'KRBTEST.COM',
20:03:45.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
20:03:45.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 20:03:45 dist-test-slave-b0rl krb5kdc[22741](info): setting up network...
20:03:45.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
20:03:45.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 20:03:45 dist-test-slave-b0rl krb5kdc[22741](info): set up 2 sockets
20:03:45.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 20:03:45 dist-test-slave-b0rl krb5kdc[22741](info): commencing operation
20:03:45.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
20:03:45.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:45.548643 22717 heartbeater.cc:499] Master 127.21.124.124:45763 was elected leader, sending a full tablet report...
20:03:47.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:47.579321 22729 mini_kdc.cc:121] Time spent starting KDC: real 2.179s user 0.000s sys 0.025s
20:03:47.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
20:03:47.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
20:03:47.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
20:03:47.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
20:03:47.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
20:03:47.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
20:03:47.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
20:03:47.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
20:03:47.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
20:03:47.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
20:03:47.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/krb5kdc/test-user.keytab.
20:03:47.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/krb5kdc/test-user.keytab.
20:03:47.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 20:03:47 dist-test-slave-b0rl krb5kdc[22741](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777838627, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:03:47.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
20:03:47.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-05-03T20:03:47Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
20:03:47.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-05-03T20:03:47Z Disabled control of system clock
20:03:47.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.22.50.126@KRBTEST.COM; defaulting to no policy
20:03:47.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:03:47.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.22.50.126@KRBTEST.COM" created.
20:03:47.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:03:47.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.22.50.126 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/krb5kdc/kudu.keytab.
20:03:47.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.22.50.126 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/krb5kdc/kudu.keytab.
20:03:47.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.22.50.126@KRBTEST.COM; defaulting to no policy
20:03:47.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:03:47.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.22.50.126@KRBTEST.COM" created.
20:03:47.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:03:47.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.22.50.126 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/krb5kdc/kudu.keytab.
20:03:47.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.22.50.126 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/krb5kdc/kudu.keytab.
20:03:47.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:47.829368 22729 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskqmogBp/build/tsan/bin/kudu
20:03:47.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskqmogBp/build/tsan/bin/kudu
20:03:47.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/master-0/wal
20:03:47.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/master-0/data
20:03:47.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:03:47.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:03:47.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:03:47.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:03:47.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:03:47.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:03:47.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/master-0/logs
20:03:47.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/master-0/data/info.pb
20:03:47.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:03:47.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:03:47.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:03:47.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:03:47.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:03:47.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:03:47.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:03:47.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:03:47.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:03:47.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:03:47.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:03:47.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:03:47.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.50.126:34829
20:03:47.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.50.126
20:03:47.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:03:47.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/krb5kdc/kudu.keytab
20:03:47.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.22.50.126
20:03:47.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:03:47.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:03:47.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:03:47.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
20:03:47.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.50.84:44541
20:03:47.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:03:47.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:03:47.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:03:47.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:03:47.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.50.126:34829,127.22.50.125:39091,127.22.50.124:43653
20:03:47.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:03:47.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:32783/jwks.json
20:03:47.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
20:03:47.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:03:48.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:48.162909 22758 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:03:48.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:48.163473 22758 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:03:48.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:48.164029 22758 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:03:48.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:48.198208 22758 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
20:03:48.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:48.198457 22758 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
20:03:48.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:48.198869 22758 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:03:48.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:48.199092 22758 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
20:03:48.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:48.199291 22758 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
20:03:48.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:48.199484 22758 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
20:03:48.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:48.199621 22758 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:32783/jwks.json
20:03:48.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:48.238421 22758 master_runner.cc:386] Master server non-default flags:
20:03:48.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:03:48.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.50.84:44541
20:03:48.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:03:48.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:03:48.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/master-0/data
20:03:48.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/master-0/wal
20:03:48.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
20:03:48.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:03:48.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.50.126:34829,127.22.50.125:39091,127.22.50.124:43653
20:03:48.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/krb5kdc/kudu.keytab
20:03:48.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.22.50.126
20:03:48.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:03:48.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:03:48.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:03:48.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.50.126:34829
20:03:48.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:03:48.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:03:48.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:03:48.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:32783/jwks.json
20:03:48.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:03:48.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:03:48.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:03:48.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:03:48.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:03:48.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/master-0/data/info.pb
20:03:48.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.50.126
20:03:48.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:03:48.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:03:48.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:03:48.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:03:48.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:03:48.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:03:48.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/master-0/logs
20:03:48.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:03:48.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:03:48.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:03:48.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:03:48.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:03:48.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0ba7f54e15c9f673bf3e350f30b8d5f1a4037b79
20:03:48.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:03:48.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 03 May 2026 19:43:25 UTC on e7f111948823
20:03:48.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11731
20:03:48.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:03:48.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:48.239737 22758 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:03:48.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:48.240780 22758 file_cache.cc:492] Constructed file cache file cache with capacity 419430
20:03:48.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:48.253803 22764 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:49.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:48.262524 22758 server_base.cc:1061] running on GCE node
20:03:49.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:48.259357 22765 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:49.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:48.260118 22767 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:49.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:49.555356 22758 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:03:49.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:49.559051 22758 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:03:49.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:49.560866 22758 hybrid_clock.cc:648] HybridClock initialized: now 1777838629560823 us; error 47 us; skew 500 ppm
20:03:49.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:49.562336 22758 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:03:49.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 20:03:49 dist-test-slave-b0rl krb5kdc[22741](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777838629, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.50.126@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:03:49.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:49.572300 22758 init.cc:377] Logged in from keytab as kudu/127.22.50.126@KRBTEST.COM (short username kudu)
20:03:49.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:49.577792 22758 webserver.cc:492] Webserver started at http://127.22.50.126:40523/ using document root <none> and password file <none>
20:03:49.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:49.578907 22758 fs_manager.cc:362] Metadata directory not provided
20:03:49.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:49.579181 22758 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:03:49.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:49.579664 22758 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
20:03:49.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:49.584131 22758 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/master-0/data/instance:
20:03:49.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "08b4dd9eba654c9e8faa59ad892a17ab"
20:03:49.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-03 20:03:49 on dist-test-slave-b0rl"
20:03:49.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:49.585296 22758 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/master-0/wal/instance:
20:03:49.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "08b4dd9eba654c9e8faa59ad892a17ab"
20:03:49.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-03 20:03:49 on dist-test-slave-b0rl"
20:03:49.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:49.593713 22758 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.005s sys 0.004s
20:03:49.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:49.599725 22777 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:03:49.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:49.601168 22758 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.000s sys 0.004s
20:03:49.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:49.601603 22758 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/master-0/data,/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/master-0/wal
20:03:49.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "08b4dd9eba654c9e8faa59ad892a17ab"
20:03:49.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-03 20:03:49 on dist-test-slave-b0rl"
20:03:49.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:49.602169 22758 fs_report.cc:389] FS layout report
20:03:49.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:03:49.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/master-0/wal
20:03:49.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/master-0/wal
20:03:49.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/master-0/data/data
20:03:49.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:03:49.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:03:49.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:03:49.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:03:49.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:03:49.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:03:49.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:03:49.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:03:49.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:03:49.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:03:49.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:03:49.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:49.666733 22758 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:03:49.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:49.673386 22758 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:03:49.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:49.673947 22758 kserver.cc:163] Server-wide thread pool size limit: 3276
20:03:49.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:49.798321 22758 rpc_server.cc:307] RPC server started. Bound to: 127.22.50.126:34829
20:03:49.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:49.798398 22828 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.22.50.126:34829 every 8 connection(s)
20:03:49.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:49.801155 22758 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/master-0/data/info.pb
20:03:49.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:49.809022 22729 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskqmogBp/build/tsan/bin/kudu as pid 22758
20:03:49.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:49.809286 22829 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
20:03:49.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:49.810071 22729 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/master-0/wal/instance
20:03:49.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:49.826767 22829 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.50.126" port: 34829 } has no permanent_uuid. Determining permanent_uuid...
20:03:49.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.22.50.125@KRBTEST.COM; defaulting to no policy
20:03:49.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:03:49.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.22.50.125@KRBTEST.COM" created.
20:03:49.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 20:03:49 dist-test-slave-b0rl krb5kdc[22741](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777838629, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.50.126@KRBTEST.COM for kudu/127.22.50.126@KRBTEST.COM
20:03:49.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:03:49.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.22.50.125 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/krb5kdc/kudu.keytab.
20:03:49.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.22.50.125 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/krb5kdc/kudu.keytab.
20:03:49.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:49.877208 22829 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.50.125" port: 39091 } has no permanent_uuid. Determining permanent_uuid...
20:03:49.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:49.879711 22779 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.22.50.125:39091: connect: Connection refused (error 111)
20:03:49.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.22.50.125@KRBTEST.COM; defaulting to no policy
20:03:49.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:49.884217 22829 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.22.50.125:39091: Network error: Client connection negotiation failed: client connection to 127.22.50.125:39091: connect: Connection refused (error 111)
20:03:49.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:03:49.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.22.50.125@KRBTEST.COM" created.
20:03:49.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:03:49.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.22.50.125 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/krb5kdc/kudu.keytab.
20:03:49.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.22.50.125 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/krb5kdc/kudu.keytab.
20:03:49.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:49.917490 22729 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskqmogBp/build/tsan/bin/kudu
20:03:49.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskqmogBp/build/tsan/bin/kudu
20:03:49.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/master-1/wal
20:03:49.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/master-1/data
20:03:49.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:03:49.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:03:49.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:03:49.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:03:49.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:03:49.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:03:49.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/master-1/logs
20:03:49.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/master-1/data/info.pb
20:03:49.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:03:49.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:03:49.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:03:49.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:03:49.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:03:49.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:03:49.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:03:49.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:03:49.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:03:49.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:03:49.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:03:49.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:03:49.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.50.125:39091
20:03:49.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.50.125
20:03:49.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:03:49.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/krb5kdc/kudu.keytab
20:03:49.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.22.50.125
20:03:49.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:03:49.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:03:49.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:03:49.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
20:03:49.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.50.84:44541
20:03:49.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:03:49.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:03:49.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:03:49.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:03:49.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.50.126:34829,127.22.50.125:39091,127.22.50.124:43653
20:03:49.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:03:49.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:32783/jwks.json
20:03:49.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
20:03:49.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:03:49.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:49.933691 22829 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.50.125" port: 39091 } attempt: 1
20:03:49.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:49.938032 22829 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.22.50.125:39091: Network error: Client connection negotiation failed: client connection to 127.22.50.125:39091: connect: Connection refused (error 111)
20:03:50.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:50.006492 22829 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.50.125" port: 39091 } attempt: 2
20:03:50.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:50.011075 22829 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.22.50.125:39091: Network error: Client connection negotiation failed: client connection to 127.22.50.125:39091: connect: Connection refused (error 111)
20:03:50.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:50.102622 22829 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.50.125" port: 39091 } attempt: 3
20:03:50.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:50.107988 22829 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.22.50.125:39091: Network error: Client connection negotiation failed: client connection to 127.22.50.125:39091: connect: Connection refused (error 111)
20:03:50.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:50.251456 22829 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.50.125" port: 39091 } attempt: 4
20:03:50.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:50.255582 22829 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.22.50.125:39091: Network error: Client connection negotiation failed: client connection to 127.22.50.125:39091: connect: Connection refused (error 111)
20:03:50.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:50.269102 22838 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:03:50.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:50.269780 22838 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:03:50.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:50.270750 22838 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:03:50.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:50.305696 22838 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
20:03:50.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:50.306023 22838 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
20:03:50.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:50.306458 22838 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:03:50.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:50.307049 22838 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
20:03:50.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:50.307407 22838 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
20:03:50.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:50.307781 22838 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
20:03:50.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:50.308058 22838 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:32783/jwks.json
20:03:50.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:50.349849 22838 master_runner.cc:386] Master server non-default flags:
20:03:50.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:03:50.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.50.84:44541
20:03:50.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:03:50.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:03:50.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/master-1/data
20:03:50.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/master-1/wal
20:03:50.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
20:03:50.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:03:50.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.50.126:34829,127.22.50.125:39091,127.22.50.124:43653
20:03:50.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/krb5kdc/kudu.keytab
20:03:50.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.22.50.125
20:03:50.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:03:50.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:03:50.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:03:50.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.50.125:39091
20:03:50.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:03:50.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:03:50.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:03:50.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:32783/jwks.json
20:03:50.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:03:50.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:03:50.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:03:50.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:03:50.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:03:50.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/master-1/data/info.pb
20:03:50.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.50.125
20:03:50.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:03:50.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:03:50.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:03:50.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:03:50.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:03:50.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:03:50.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/master-1/logs
20:03:50.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:03:50.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:03:50.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:03:50.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:03:50.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:03:50.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0ba7f54e15c9f673bf3e350f30b8d5f1a4037b79
20:03:50.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:03:50.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 03 May 2026 19:43:25 UTC on e7f111948823
20:03:50.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11731
20:03:50.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:03:50.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:50.352097 22838 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:03:50.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:50.353296 22838 file_cache.cc:492] Constructed file cache file cache with capacity 419430
20:03:50.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:50.365288 22845 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:50.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:50.555219 22829 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.50.125" port: 39091 } attempt: 5
20:03:50.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:50.569085 22829 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.22.50.125:39091: Network error: Client connection negotiation failed: client connection to 127.22.50.125:39091: connect: Connection refused (error 111)
20:03:51.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:51.116647 22829 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.50.125" port: 39091 } attempt: 6
20:03:51.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:51.130051 22829 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.22.50.125:39091: Network error: Client connection negotiation failed: client connection to 127.22.50.125:39091: connect: Connection refused (error 111)
20:03:51.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:50.369871 22848 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:51.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:50.365856 22846 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:51.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:51.750567 22847 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1380 milliseconds
20:03:51.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:51.750749 22838 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:03:51.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:51.752106 22838 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:03:51.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:51.755101 22838 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:03:51.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:51.756525 22838 hybrid_clock.cc:648] HybridClock initialized: now 1777838631756450 us; error 107 us; skew 500 ppm
20:03:51.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:51.757534 22838 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:03:51.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 20:03:51 dist-test-slave-b0rl krb5kdc[22741](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777838631, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.50.125@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:03:51.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:51.763918 22838 init.cc:377] Logged in from keytab as kudu/127.22.50.125@KRBTEST.COM (short username kudu)
20:03:51.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:51.770145 22838 webserver.cc:492] Webserver started at http://127.22.50.125:40527/ using document root <none> and password file <none>
20:03:51.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:51.771193 22838 fs_manager.cc:362] Metadata directory not provided
20:03:51.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:51.771450 22838 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:03:51.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:51.772018 22838 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
20:03:51.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:51.776664 22838 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/master-1/data/instance:
20:03:51.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5c356dfb31e749c5a0f704baa1957a47"
20:03:51.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-03 20:03:51 on dist-test-slave-b0rl"
20:03:51.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:51.777853 22838 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/master-1/wal/instance:
20:03:51.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5c356dfb31e749c5a0f704baa1957a47"
20:03:51.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-03 20:03:51 on dist-test-slave-b0rl"
20:03:51.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:51.786113 22838 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.009s sys 0.001s
20:03:51.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:51.792688 22859 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:03:51.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:51.793880 22838 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.004s
20:03:51.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:51.794293 22838 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/master-1/data,/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/master-1/wal
20:03:51.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5c356dfb31e749c5a0f704baa1957a47"
20:03:51.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-03 20:03:51 on dist-test-slave-b0rl"
20:03:51.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:51.794889 22838 fs_report.cc:389] FS layout report
20:03:51.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:03:51.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/master-1/wal
20:03:51.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/master-1/wal
20:03:51.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/master-1/data/data
20:03:51.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:03:51.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:03:51.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:03:51.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:03:51.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:03:51.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:03:51.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:03:51.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:03:51.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:03:51.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:03:51.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:03:51.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:51.863188 22838 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:03:51.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:51.873521 22838 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:03:51.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:51.874150 22838 kserver.cc:163] Server-wide thread pool size limit: 3276
20:03:51.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:51.964248 22838 rpc_server.cc:307] RPC server started. Bound to: 127.22.50.125:39091
20:03:51.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:51.964301 22910 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.22.50.125:39091 every 8 connection(s)
20:03:51.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:51.966960 22838 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/master-1/data/info.pb
20:03:51.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:51.970414 22729 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskqmogBp/build/tsan/bin/kudu as pid 22838
20:03:51.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:51.970928 22729 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/master-1/wal/instance
20:03:51.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:51.975184 22911 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
20:03:51.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.22.50.124@KRBTEST.COM; defaulting to no policy
20:03:51.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:51.988198 22911 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.50.126" port: 34829 } has no permanent_uuid. Determining permanent_uuid...
20:03:51.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:03:51.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.22.50.124@KRBTEST.COM" created.
20:03:52.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 20:03:52 dist-test-slave-b0rl krb5kdc[22741](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777838631, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.50.125@KRBTEST.COM for kudu/127.22.50.126@KRBTEST.COM
20:03:52.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:03:52.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.22.50.124 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/krb5kdc/kudu.keytab.
20:03:52.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.22.50.124 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/krb5kdc/kudu.keytab.
20:03:52.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:52.032752 22911 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.50.125" port: 39091 } has no permanent_uuid. Determining permanent_uuid...
20:03:52.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.22.50.124@KRBTEST.COM; defaulting to no policy
20:03:52.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:03:52.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.22.50.124@KRBTEST.COM" created.
20:03:52.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 20:03:52 dist-test-slave-b0rl krb5kdc[22741](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777838631, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.50.125@KRBTEST.COM for kudu/127.22.50.125@KRBTEST.COM
20:03:52.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:52.064918 22911 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.50.124" port: 43653 } has no permanent_uuid. Determining permanent_uuid...
20:03:52.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:52.067400 22862 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.22.50.124:43653: connect: Connection refused (error 111)
20:03:52.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:52.072077 22911 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.22.50.124:43653: Network error: Client connection negotiation failed: client connection to 127.22.50.124:43653: connect: Connection refused (error 111)
20:03:52.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:03:52.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.22.50.124 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/krb5kdc/kudu.keytab.
20:03:52.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.22.50.124 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/krb5kdc/kudu.keytab.
20:03:52.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:52.076363 22729 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskqmogBp/build/tsan/bin/kudu
20:03:52.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskqmogBp/build/tsan/bin/kudu
20:03:52.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/master-2/wal
20:03:52.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/master-2/data
20:03:52.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:03:52.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:03:52.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:03:52.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:03:52.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:03:52.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:03:52.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/master-2/logs
20:03:52.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/master-2/data/info.pb
20:03:52.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:03:52.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:03:52.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:03:52.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:03:52.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:03:52.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:03:52.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:03:52.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:03:52.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:03:52.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:03:52.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:03:52.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:03:52.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.50.124:43653
20:03:52.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.50.124
20:03:52.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:03:52.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/krb5kdc/kudu.keytab
20:03:52.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.22.50.124
20:03:52.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:03:52.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:03:52.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:03:52.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
20:03:52.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.50.84:44541
20:03:52.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:03:52.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:03:52.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:03:52.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:03:52.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.50.126:34829,127.22.50.125:39091,127.22.50.124:43653
20:03:52.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:03:52.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:32783/jwks.json
20:03:52.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
20:03:52.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:03:52.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:52.121580 22911 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.50.124" port: 43653 } attempt: 1
20:03:52.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:52.126080 22911 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.22.50.124:43653: Network error: Client connection negotiation failed: client connection to 127.22.50.124:43653: connect: Connection refused (error 111)
20:03:52.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:52.190657 22829 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.50.125" port: 39091 } attempt: 7
20:03:52.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:52.194528 22911 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.50.124" port: 43653 } attempt: 2
20:03:52.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:52.199707 22911 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.22.50.124:43653: Network error: Client connection negotiation failed: client connection to 127.22.50.124:43653: connect: Connection refused (error 111)
20:03:52.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 20:03:52 dist-test-slave-b0rl krb5kdc[22741](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777838629, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.50.126@KRBTEST.COM for kudu/127.22.50.125@KRBTEST.COM
20:03:52.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:52.217998 22829 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.50.124" port: 43653 } has no permanent_uuid. Determining permanent_uuid...
20:03:52.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:52.224591 22829 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.22.50.124:43653: Network error: Client connection negotiation failed: client connection to 127.22.50.124:43653: connect: Connection refused (error 111)
20:03:52.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:52.283082 22829 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.50.124" port: 43653 } attempt: 1
20:03:52.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:52.287797 22829 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.22.50.124:43653: Network error: Client connection negotiation failed: client connection to 127.22.50.124:43653: connect: Connection refused (error 111)
20:03:52.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:52.291162 22911 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.50.124" port: 43653 } attempt: 3
20:03:52.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:52.296051 22911 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.22.50.124:43653: Network error: Client connection negotiation failed: client connection to 127.22.50.124:43653: connect: Connection refused (error 111)
20:03:52.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:52.369186 22829 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.50.124" port: 43653 } attempt: 2
20:03:52.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:52.373368 22829 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.22.50.124:43653: Network error: Client connection negotiation failed: client connection to 127.22.50.124:43653: connect: Connection refused (error 111)
20:03:52.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:52.434981 22921 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:03:52.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:52.435540 22921 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:03:52.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:52.436100 22921 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:03:52.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:52.439702 22911 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.50.124" port: 43653 } attempt: 4
20:03:52.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:52.445072 22911 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.22.50.124:43653: Network error: Client connection negotiation failed: client connection to 127.22.50.124:43653: connect: Connection refused (error 111)
20:03:52.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:52.458781 22829 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.50.124" port: 43653 } attempt: 3
20:03:52.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:52.462687 22829 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.22.50.124:43653: Network error: Client connection negotiation failed: client connection to 127.22.50.124:43653: connect: Connection refused (error 111)
20:03:52.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:52.474303 22921 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
20:03:52.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:52.474642 22921 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
20:03:52.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:52.474980 22921 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:03:52.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:52.475186 22921 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
20:03:52.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:52.475386 22921 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
20:03:52.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:52.475587 22921 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
20:03:52.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:52.475737 22921 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:32783/jwks.json
20:03:52.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:52.523579 22921 master_runner.cc:386] Master server non-default flags:
20:03:52.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:03:52.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.50.84:44541
20:03:52.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:03:52.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:03:52.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/master-2/data
20:03:52.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/master-2/wal
20:03:52.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
20:03:52.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:03:52.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.50.126:34829,127.22.50.125:39091,127.22.50.124:43653
20:03:52.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/krb5kdc/kudu.keytab
20:03:52.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.22.50.124
20:03:52.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:03:52.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:03:52.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:03:52.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.50.124:43653
20:03:52.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:03:52.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:03:52.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:03:52.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:32783/jwks.json
20:03:52.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:03:52.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:03:52.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:03:52.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:03:52.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:03:52.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/master-2/data/info.pb
20:03:52.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.50.124
20:03:52.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:03:52.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:03:52.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:03:52.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:03:52.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:03:52.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:03:52.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/master-2/logs
20:03:52.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:03:52.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:03:52.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:03:52.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:03:52.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:03:52.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0ba7f54e15c9f673bf3e350f30b8d5f1a4037b79
20:03:52.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:03:52.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 03 May 2026 19:43:25 UTC on e7f111948823
20:03:52.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11731
20:03:52.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:03:52.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:52.524963 22921 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:03:52.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:52.526016 22921 file_cache.cc:492] Constructed file cache file cache with capacity 419430
20:03:52.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:52.539340 22929 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:52.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:52.603199 22829 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.50.124" port: 43653 } attempt: 4
20:03:52.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:52.610944 22829 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.22.50.124:43653: Network error: Client connection negotiation failed: client connection to 127.22.50.124:43653: connect: Connection refused (error 111)
20:03:52.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:52.744495 22911 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.50.124" port: 43653 } attempt: 5
20:03:52.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:52.751785 22911 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.22.50.124:43653: Network error: Client connection negotiation failed: client connection to 127.22.50.124:43653: connect: Connection refused (error 111)
20:03:52.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:52.894529 22829 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.50.124" port: 43653 } attempt: 5
20:03:52.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:52.908327 22829 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.22.50.124:43653: Network error: Client connection negotiation failed: client connection to 127.22.50.124:43653: connect: Connection refused (error 111)
20:03:53.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:53.299419 22911 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.50.124" port: 43653 } attempt: 6
20:03:53.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:53.339650 22911 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.22.50.124:43653: Network error: Client connection negotiation failed: client connection to 127.22.50.124:43653: connect: Connection refused (error 111)
20:03:53.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:53.460904 22829 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.50.124" port: 43653 } attempt: 6
20:03:53.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:53.496202 22829 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.22.50.124:43653: Network error: Client connection negotiation failed: client connection to 127.22.50.124:43653: connect: Connection refused (error 111)
20:03:54.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:53.941802 22928 debug-util.cc:398] Leaking SignalData structure 0x7b0800028d00 after lost signal to thread 22921
20:03:54.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:54.275249 22921 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.735s user 0.583s sys 1.003s
20:03:54.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:54.275727 22921 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.735s user 0.583s sys 1.003s
20:03:54.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:52.541002 22930 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:54.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:54.279963 22932 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:54.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:54.287528 22931 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1737 milliseconds
20:03:54.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:54.287624 22921 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:03:54.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:54.289057 22921 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:03:54.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:54.291668 22921 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:03:54.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:54.293228 22921 hybrid_clock.cc:648] HybridClock initialized: now 1777838634293178 us; error 66 us; skew 500 ppm
20:03:54.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:54.294569 22921 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:03:54.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 20:03:54 dist-test-slave-b0rl krb5kdc[22741](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777838634, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.50.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:03:54.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:54.304662 22921 init.cc:377] Logged in from keytab as kudu/127.22.50.124@KRBTEST.COM (short username kudu)
20:03:54.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:54.310508 22921 webserver.cc:492] Webserver started at http://127.22.50.124:36701/ using document root <none> and password file <none>
20:03:54.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:54.311488 22921 fs_manager.cc:362] Metadata directory not provided
20:03:54.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:54.311688 22921 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:03:54.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:54.312198 22921 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
20:03:54.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:54.316823 22921 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/master-2/data/instance:
20:03:54.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7d8f4ab9d826436490a94ee04f174e7f"
20:03:54.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-03 20:03:54 on dist-test-slave-b0rl"
20:03:54.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:54.318037 22921 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/master-2/wal/instance:
20:03:54.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7d8f4ab9d826436490a94ee04f174e7f"
20:03:54.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-03 20:03:54 on dist-test-slave-b0rl"
20:03:54.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:54.327142 22921 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.002s sys 0.005s
20:03:54.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:54.337041 22948 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:03:54.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:54.339072 22921 fs_manager.cc:730] Time spent opening block manager: real 0.006s user 0.006s sys 0.001s
20:03:54.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:54.339607 22921 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/master-2/data,/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/master-2/wal
20:03:54.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7d8f4ab9d826436490a94ee04f174e7f"
20:03:54.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-03 20:03:54 on dist-test-slave-b0rl"
20:03:54.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:54.340006 22921 fs_report.cc:389] FS layout report
20:03:54.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:03:54.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/master-2/wal
20:03:54.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/master-2/wal
20:03:54.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/master-2/data/data
20:03:54.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:03:54.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:03:54.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:03:54.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:03:54.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:03:54.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:03:54.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:03:54.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:03:54.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:03:54.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:03:54.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:03:54.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:54.400429 22911 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.50.124" port: 43653 } attempt: 7
20:03:54.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:54.408806 22911 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.22.50.124:43653: Network error: Client connection negotiation failed: client connection to 127.22.50.124:43653: connect: Connection refused (error 111)
20:03:54.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:54.420531 22921 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:03:54.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:54.446996 22921 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:03:54.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:54.449038 22921 kserver.cc:163] Server-wide thread pool size limit: 3276
20:03:54.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:54.529964 22829 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.50.124" port: 43653 } attempt: 7
20:03:54.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:54.550211 22829 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.22.50.124:43653: Network error: Client connection negotiation failed: client connection to 127.22.50.124:43653: connect: Connection refused (error 111)
20:03:54.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:54.587460 22921 rpc_server.cc:307] RPC server started. Bound to: 127.22.50.124:43653
20:03:54.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:54.588421 23003 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.22.50.124:43653 every 8 connection(s)
20:03:54.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:54.591917 22921 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/master-2/data/info.pb
20:03:54.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:54.599810 23004 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
20:03:54.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:54.600385 22729 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskqmogBp/build/tsan/bin/kudu as pid 22921
20:03:54.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:54.600836 22729 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/master-2/wal/instance
20:03:54.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:54.619464 23004 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.50.126" port: 34829 } has no permanent_uuid. Determining permanent_uuid...
20:03:54.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 20:03:54 dist-test-slave-b0rl krb5kdc[22741](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777838627, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.22.50.126@KRBTEST.COM
20:03:54.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 20:03:54 dist-test-slave-b0rl krb5kdc[22741](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777838627, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.22.50.125@KRBTEST.COM
20:03:54.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 20:03:54 dist-test-slave-b0rl krb5kdc[22741](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777838634, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.50.124@KRBTEST.COM for kudu/127.22.50.126@KRBTEST.COM
20:03:54.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 20:03:54 dist-test-slave-b0rl krb5kdc[22741](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777838627, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.22.50.124@KRBTEST.COM
20:03:54.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:54.725418 23004 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.50.125" port: 39091 } has no permanent_uuid. Determining permanent_uuid...
20:03:54.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 20:03:54 dist-test-slave-b0rl krb5kdc[22741](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777838634, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.50.124@KRBTEST.COM for kudu/127.22.50.125@KRBTEST.COM
20:03:54.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:54.751282 23004 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.50.124" port: 43653 } has no permanent_uuid. Determining permanent_uuid...
20:03:54.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 20:03:54 dist-test-slave-b0rl krb5kdc[22741](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777838634, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.50.124@KRBTEST.COM for kudu/127.22.50.124@KRBTEST.COM
20:03:54.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:54.784955 23004 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 7d8f4ab9d826436490a94ee04f174e7f: Bootstrap starting.
20:03:54.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:54.791237 23004 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 7d8f4ab9d826436490a94ee04f174e7f: Neither blocks nor log segments found. Creating new log.
20:03:54.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:54.794230 23004 log.cc:826] T 00000000000000000000000000000000 P 7d8f4ab9d826436490a94ee04f174e7f: Log is configured to *not* fsync() on all Append() calls
20:03:54.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:54.800558 23004 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 7d8f4ab9d826436490a94ee04f174e7f: No bootstrap required, opened a new log
20:03:54.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:54.821911 23004 raft_consensus.cc:359] T 00000000000000000000000000000000 P 7d8f4ab9d826436490a94ee04f174e7f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "08b4dd9eba654c9e8faa59ad892a17ab" member_type: VOTER last_known_addr { host: "127.22.50.126" port: 34829 } } peers { permanent_uuid: "5c356dfb31e749c5a0f704baa1957a47" member_type: VOTER last_known_addr { host: "127.22.50.125" port: 39091 } } peers { permanent_uuid: "7d8f4ab9d826436490a94ee04f174e7f" member_type: VOTER last_known_addr { host: "127.22.50.124" port: 43653 } }
20:03:54.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:54.822638 23004 raft_consensus.cc:385] T 00000000000000000000000000000000 P 7d8f4ab9d826436490a94ee04f174e7f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:03:54.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:54.822916 23004 raft_consensus.cc:740] T 00000000000000000000000000000000 P 7d8f4ab9d826436490a94ee04f174e7f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7d8f4ab9d826436490a94ee04f174e7f, State: Initialized, Role: FOLLOWER
20:03:54.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:54.823764 23004 consensus_queue.cc:260] T 00000000000000000000000000000000 P 7d8f4ab9d826436490a94ee04f174e7f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "08b4dd9eba654c9e8faa59ad892a17ab" member_type: VOTER last_known_addr { host: "127.22.50.126" port: 34829 } } peers { permanent_uuid: "5c356dfb31e749c5a0f704baa1957a47" member_type: VOTER last_known_addr { host: "127.22.50.125" port: 39091 } } peers { permanent_uuid: "7d8f4ab9d826436490a94ee04f174e7f" member_type: VOTER last_known_addr { host: "127.22.50.124" port: 43653 } }
20:03:54.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:54.826970 23013 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7d8f4ab9d826436490a94ee04f174e7f [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "08b4dd9eba654c9e8faa59ad892a17ab" member_type: VOTER last_known_addr { host: "127.22.50.126" port: 34829 } } peers { permanent_uuid: "5c356dfb31e749c5a0f704baa1957a47" member_type: VOTER last_known_addr { host: "127.22.50.125" port: 39091 } } peers { permanent_uuid: "7d8f4ab9d826436490a94ee04f174e7f" member_type: VOTER last_known_addr { host: "127.22.50.124" port: 43653 } } }
20:03:54.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:54.827716 23013 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7d8f4ab9d826436490a94ee04f174e7f [sys.catalog]: This master's current role is: FOLLOWER
20:03:54.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:54.828327 23004 sys_catalog.cc:565] T 00000000000000000000000000000000 P 7d8f4ab9d826436490a94ee04f174e7f [sys.catalog]: configured and running, proceeding with master startup.
20:03:54.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:54.861008 23024 catalog_manager.cc:1568] T 00000000000000000000000000000000 P 7d8f4ab9d826436490a94ee04f174e7f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:03:54.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:54.861804 23024 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:03:54.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:54.856788 22921 master_runner.cc:186] Error getting master registration for 127.22.50.126:34829: OK, instance_id { permanent_uuid: "08b4dd9eba654c9e8faa59ad892a17ab" instance_seqno: 1777838629770925 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
20:03:54.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:54.883051 22921 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:03:54.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:54.959436 22956 tablet.cc:2404] T 00000000000000000000000000000000 P 7d8f4ab9d826436490a94ee04f174e7f: Can't schedule compaction. Clean time has not been advanced past its initial value.
20:03:55.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:55.118690 23013 raft_consensus.cc:493] T 00000000000000000000000000000000 P 7d8f4ab9d826436490a94ee04f174e7f [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:03:55.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:55.119300 23013 raft_consensus.cc:515] T 00000000000000000000000000000000 P 7d8f4ab9d826436490a94ee04f174e7f [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "08b4dd9eba654c9e8faa59ad892a17ab" member_type: VOTER last_known_addr { host: "127.22.50.126" port: 34829 } } peers { permanent_uuid: "5c356dfb31e749c5a0f704baa1957a47" member_type: VOTER last_known_addr { host: "127.22.50.125" port: 39091 } } peers { permanent_uuid: "7d8f4ab9d826436490a94ee04f174e7f" member_type: VOTER last_known_addr { host: "127.22.50.124" port: 43653 } }
20:03:55.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:55.122030 23013 leader_election.cc:290] T 00000000000000000000000000000000 P 7d8f4ab9d826436490a94ee04f174e7f [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 08b4dd9eba654c9e8faa59ad892a17ab (127.22.50.126:34829), 5c356dfb31e749c5a0f704baa1957a47 (127.22.50.125:39091)
20:03:55.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:55.683470 22152 debug-util.cc:398] Leaking SignalData structure 0x7b0800080b40 after lost signal to thread 22087
20:03:55.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:55.684705 22152 debug-util.cc:398] Leaking SignalData structure 0x7b08000957e0 after lost signal to thread 22155
20:03:55.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:55.869586 23024 catalog_manager.cc:1568] T 00000000000000000000000000000000 P 7d8f4ab9d826436490a94ee04f174e7f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:03:55.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:55.870061 23024 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:03:56.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:56.499228 22911 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.50.124" port: 43653 } attempt: 8
20:03:56.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 20:03:56 dist-test-slave-b0rl krb5kdc[22741](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777838631, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.50.125@KRBTEST.COM for kudu/127.22.50.124@KRBTEST.COM
20:03:56.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:56.547232 22911 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5c356dfb31e749c5a0f704baa1957a47: Bootstrap starting.
20:03:56.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:56.555334 22911 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 5c356dfb31e749c5a0f704baa1957a47: Neither blocks nor log segments found. Creating new log.
20:03:56.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:56.559188 22911 log.cc:826] T 00000000000000000000000000000000 P 5c356dfb31e749c5a0f704baa1957a47: Log is configured to *not* fsync() on all Append() calls
20:03:56.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:56.567265 22911 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5c356dfb31e749c5a0f704baa1957a47: No bootstrap required, opened a new log
20:03:56.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:56.607142 22911 raft_consensus.cc:359] T 00000000000000000000000000000000 P 5c356dfb31e749c5a0f704baa1957a47 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "08b4dd9eba654c9e8faa59ad892a17ab" member_type: VOTER last_known_addr { host: "127.22.50.126" port: 34829 } } peers { permanent_uuid: "5c356dfb31e749c5a0f704baa1957a47" member_type: VOTER last_known_addr { host: "127.22.50.125" port: 39091 } } peers { permanent_uuid: "7d8f4ab9d826436490a94ee04f174e7f" member_type: VOTER last_known_addr { host: "127.22.50.124" port: 43653 } }
20:03:56.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:56.608237 22911 raft_consensus.cc:385] T 00000000000000000000000000000000 P 5c356dfb31e749c5a0f704baa1957a47 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:03:56.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:56.608546 22911 raft_consensus.cc:740] T 00000000000000000000000000000000 P 5c356dfb31e749c5a0f704baa1957a47 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5c356dfb31e749c5a0f704baa1957a47, State: Initialized, Role: FOLLOWER
20:03:56.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:56.609524 22911 consensus_queue.cc:260] T 00000000000000000000000000000000 P 5c356dfb31e749c5a0f704baa1957a47 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "08b4dd9eba654c9e8faa59ad892a17ab" member_type: VOTER last_known_addr { host: "127.22.50.126" port: 34829 } } peers { permanent_uuid: "5c356dfb31e749c5a0f704baa1957a47" member_type: VOTER last_known_addr { host: "127.22.50.125" port: 39091 } } peers { permanent_uuid: "7d8f4ab9d826436490a94ee04f174e7f" member_type: VOTER last_known_addr { host: "127.22.50.124" port: 43653 } }
20:03:56.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:56.611680 22829 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.50.124" port: 43653 } attempt: 8
20:03:56.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:56.626148 23032 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5c356dfb31e749c5a0f704baa1957a47 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "08b4dd9eba654c9e8faa59ad892a17ab" member_type: VOTER last_known_addr { host: "127.22.50.126" port: 34829 } } peers { permanent_uuid: "5c356dfb31e749c5a0f704baa1957a47" member_type: VOTER last_known_addr { host: "127.22.50.125" port: 39091 } } peers { permanent_uuid: "7d8f4ab9d826436490a94ee04f174e7f" member_type: VOTER last_known_addr { host: "127.22.50.124" port: 43653 } } }
20:03:56.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:56.628031 23032 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5c356dfb31e749c5a0f704baa1957a47 [sys.catalog]: This master's current role is: FOLLOWER
20:03:56.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:56.629168 23032 raft_consensus.cc:493] T 00000000000000000000000000000000 P 5c356dfb31e749c5a0f704baa1957a47 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:03:56.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:56.633996 23032 raft_consensus.cc:515] T 00000000000000000000000000000000 P 5c356dfb31e749c5a0f704baa1957a47 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "08b4dd9eba654c9e8faa59ad892a17ab" member_type: VOTER last_known_addr { host: "127.22.50.126" port: 34829 } } peers { permanent_uuid: "5c356dfb31e749c5a0f704baa1957a47" member_type: VOTER last_known_addr { host: "127.22.50.125" port: 39091 } } peers { permanent_uuid: "7d8f4ab9d826436490a94ee04f174e7f" member_type: VOTER last_known_addr { host: "127.22.50.124" port: 43653 } }
20:03:56.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:56.642458 23032 leader_election.cc:290] T 00000000000000000000000000000000 P 5c356dfb31e749c5a0f704baa1957a47 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 08b4dd9eba654c9e8faa59ad892a17ab (127.22.50.126:34829), 7d8f4ab9d826436490a94ee04f174e7f (127.22.50.124:43653)
20:03:56.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:56.643317 22977 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "5c356dfb31e749c5a0f704baa1957a47" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7d8f4ab9d826436490a94ee04f174e7f" is_pre_election: true
20:03:56.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:56.644057 22977 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 7d8f4ab9d826436490a94ee04f174e7f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 5c356dfb31e749c5a0f704baa1957a47 in term 0.
20:03:56.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:56.650334 22862 leader_election.cc:304] T 00000000000000000000000000000000 P 5c356dfb31e749c5a0f704baa1957a47 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 5c356dfb31e749c5a0f704baa1957a47, 7d8f4ab9d826436490a94ee04f174e7f; no voters:
20:03:56.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:56.652395 23032 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 5c356dfb31e749c5a0f704baa1957a47 [term 0 FOLLOWER]: Leader pre-election won for term 1
20:03:56.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:56.657770 23032 raft_consensus.cc:493] T 00000000000000000000000000000000 P 5c356dfb31e749c5a0f704baa1957a47 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
20:03:56.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:56.658169 23032 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 5c356dfb31e749c5a0f704baa1957a47 [term 0 FOLLOWER]: Advancing to term 1
20:03:56.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 20:03:56 dist-test-slave-b0rl krb5kdc[22741](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777838629, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.50.126@KRBTEST.COM for kudu/127.22.50.124@KRBTEST.COM
20:03:56.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:56.666383 22911 sys_catalog.cc:565] T 00000000000000000000000000000000 P 5c356dfb31e749c5a0f704baa1957a47 [sys.catalog]: configured and running, proceeding with master startup.
20:03:56.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:56.666544 22886 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "7d8f4ab9d826436490a94ee04f174e7f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5c356dfb31e749c5a0f704baa1957a47" is_pre_election: true
20:03:56.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:56.669467 23032 raft_consensus.cc:515] T 00000000000000000000000000000000 P 5c356dfb31e749c5a0f704baa1957a47 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "08b4dd9eba654c9e8faa59ad892a17ab" member_type: VOTER last_known_addr { host: "127.22.50.126" port: 34829 } } peers { permanent_uuid: "5c356dfb31e749c5a0f704baa1957a47" member_type: VOTER last_known_addr { host: "127.22.50.125" port: 39091 } } peers { permanent_uuid: "7d8f4ab9d826436490a94ee04f174e7f" member_type: VOTER last_known_addr { host: "127.22.50.124" port: 43653 } }
20:03:56.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:56.672199 23032 leader_election.cc:290] T 00000000000000000000000000000000 P 5c356dfb31e749c5a0f704baa1957a47 [CANDIDATE]: Term 1 election: Requested vote from peers 08b4dd9eba654c9e8faa59ad892a17ab (127.22.50.126:34829), 7d8f4ab9d826436490a94ee04f174e7f (127.22.50.124:43653)
20:03:56.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:56.673158 22977 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "5c356dfb31e749c5a0f704baa1957a47" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7d8f4ab9d826436490a94ee04f174e7f"
20:03:56.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:56.673923 22977 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 7d8f4ab9d826436490a94ee04f174e7f [term 0 FOLLOWER]: Advancing to term 1
20:03:56.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:56.674338 22886 raft_consensus.cc:2393] T 00000000000000000000000000000000 P 5c356dfb31e749c5a0f704baa1957a47 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 7d8f4ab9d826436490a94ee04f174e7f in current term 1: Already voted for candidate 5c356dfb31e749c5a0f704baa1957a47 in this term.
20:03:56.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:56.681660 22977 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 7d8f4ab9d826436490a94ee04f174e7f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 5c356dfb31e749c5a0f704baa1957a47 in term 1.
20:03:56.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:56.682886 22862 leader_election.cc:304] T 00000000000000000000000000000000 P 5c356dfb31e749c5a0f704baa1957a47 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 5c356dfb31e749c5a0f704baa1957a47, 7d8f4ab9d826436490a94ee04f174e7f; no voters:
20:03:56.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:56.683979 23032 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 5c356dfb31e749c5a0f704baa1957a47 [term 1 FOLLOWER]: Leader election won for term 1
20:03:56.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:56.684780 23032 raft_consensus.cc:697] T 00000000000000000000000000000000 P 5c356dfb31e749c5a0f704baa1957a47 [term 1 LEADER]: Becoming Leader. State: Replica: 5c356dfb31e749c5a0f704baa1957a47, State: Running, Role: LEADER
20:03:56.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:56.685935 23032 consensus_queue.cc:237] T 00000000000000000000000000000000 P 5c356dfb31e749c5a0f704baa1957a47 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "08b4dd9eba654c9e8faa59ad892a17ab" member_type: VOTER last_known_addr { host: "127.22.50.126" port: 34829 } } peers { permanent_uuid: "5c356dfb31e749c5a0f704baa1957a47" member_type: VOTER last_known_addr { host: "127.22.50.125" port: 39091 } } peers { permanent_uuid: "7d8f4ab9d826436490a94ee04f174e7f" member_type: VOTER last_known_addr { host: "127.22.50.124" port: 43653 } }
20:03:56.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:56.697325 22886 rpcz_store.cc:267] Call kudu.consensus.ConsensusService.RequestConsensusVote from 127.0.0.1:45570 (request call id 1) took 1574 ms (client timeout 1710 ms). Trace:
20:03:56.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:56.701683 22886 rpcz_store.cc:269] 0503 20:03:55.122940 (+ 0us) service_pool.cc:167] Inserting onto call queue
20:03:56.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 20:03:55.123101 (+ 161us) service_pool.cc:224] Handling call
20:03:56.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 20:03:56.696116 (+1573015us) spinlock_profiling.cc:243] Waited 6.54 ms on lock 0x7b5400030554. stack: 00007f48f9da8c83 00007f48f9da8b80 00007f48f9919574 000000000050e094 000000000050e029 00007f4900e00951 00007f4900e9b6da 00007f4901eca1d7 00007f490057e834 00007f490057e7c2 00007f490057e6f1 00007f490057e66c 00007f490057d922 00007f48feb8abcc 00007f48feb8a006 00007f48feb899af
20:03:56.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 20:03:56.696855 (+ 739us) inbound_call.cc:177] Queueing success response
20:03:56.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"spinlock_wait_cycles":15035776}
20:03:56.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:56.725010 23031 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5c356dfb31e749c5a0f704baa1957a47 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 5c356dfb31e749c5a0f704baa1957a47. Latest consensus state: current_term: 1 leader_uuid: "5c356dfb31e749c5a0f704baa1957a47" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "08b4dd9eba654c9e8faa59ad892a17ab" member_type: VOTER last_known_addr { host: "127.22.50.126" port: 34829 } } peers { permanent_uuid: "5c356dfb31e749c5a0f704baa1957a47" member_type: VOTER last_known_addr { host: "127.22.50.125" port: 39091 } } peers { permanent_uuid: "7d8f4ab9d826436490a94ee04f174e7f" member_type: VOTER last_known_addr { host: "127.22.50.124" port: 43653 } } }
20:03:56.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:56.725876 23031 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5c356dfb31e749c5a0f704baa1957a47 [sys.catalog]: This master's current role is: LEADER
20:03:56.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:56.732180 22829 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 08b4dd9eba654c9e8faa59ad892a17ab: Bootstrap starting.
20:03:56.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:56.740857 22829 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 08b4dd9eba654c9e8faa59ad892a17ab: Neither blocks nor log segments found. Creating new log.
20:03:56.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:56.743765 22829 log.cc:826] T 00000000000000000000000000000000 P 08b4dd9eba654c9e8faa59ad892a17ab: Log is configured to *not* fsync() on all Append() calls
20:03:56.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:56.750663 23038 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
20:03:56.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:56.798832 22829 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 08b4dd9eba654c9e8faa59ad892a17ab: No bootstrap required, opened a new log
20:03:56.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:56.833177 22950 proxy.cc:239] Call had error, refreshing address and retrying: Timed out: RequestConsensusVote RPC to 127.22.50.126:34829 timed out after 1.710s (SENT)
20:03:56.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:56.862437 23038 catalog_manager.cc:1494] Initializing Kudu cluster ID...
20:03:56.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:56.864284 22829 raft_consensus.cc:359] T 00000000000000000000000000000000 P 08b4dd9eba654c9e8faa59ad892a17ab [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "08b4dd9eba654c9e8faa59ad892a17ab" member_type: VOTER last_known_addr { host: "127.22.50.126" port: 34829 } } peers { permanent_uuid: "5c356dfb31e749c5a0f704baa1957a47" member_type: VOTER last_known_addr { host: "127.22.50.125" port: 39091 } } peers { permanent_uuid: "7d8f4ab9d826436490a94ee04f174e7f" member_type: VOTER last_known_addr { host: "127.22.50.124" port: 43653 } }
20:03:56.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:56.868810 22829 raft_consensus.cc:385] T 00000000000000000000000000000000 P 08b4dd9eba654c9e8faa59ad892a17ab [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:03:56.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:56.870831 22829 raft_consensus.cc:740] T 00000000000000000000000000000000 P 08b4dd9eba654c9e8faa59ad892a17ab [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 08b4dd9eba654c9e8faa59ad892a17ab, State: Initialized, Role: FOLLOWER
20:03:56.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:56.875278 23024 catalog_manager.cc:1568] T 00000000000000000000000000000000 P 7d8f4ab9d826436490a94ee04f174e7f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:03:56.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:56.875577 23024 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:03:56.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:56.874339 22829 consensus_queue.cc:260] T 00000000000000000000000000000000 P 08b4dd9eba654c9e8faa59ad892a17ab [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "08b4dd9eba654c9e8faa59ad892a17ab" member_type: VOTER last_known_addr { host: "127.22.50.126" port: 34829 } } peers { permanent_uuid: "5c356dfb31e749c5a0f704baa1957a47" member_type: VOTER last_known_addr { host: "127.22.50.125" port: 39091 } } peers { permanent_uuid: "7d8f4ab9d826436490a94ee04f174e7f" member_type: VOTER last_known_addr { host: "127.22.50.124" port: 43653 } }
20:03:56.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:56.887704 22865 tablet.cc:2404] T 00000000000000000000000000000000 P 5c356dfb31e749c5a0f704baa1957a47: Can't schedule compaction. Clean time has not been advanced past its initial value.
20:03:56.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:56.908793 23050 sys_catalog.cc:455] T 00000000000000000000000000000000 P 08b4dd9eba654c9e8faa59ad892a17ab [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "08b4dd9eba654c9e8faa59ad892a17ab" member_type: VOTER last_known_addr { host: "127.22.50.126" port: 34829 } } peers { permanent_uuid: "5c356dfb31e749c5a0f704baa1957a47" member_type: VOTER last_known_addr { host: "127.22.50.125" port: 39091 } } peers { permanent_uuid: "7d8f4ab9d826436490a94ee04f174e7f" member_type: VOTER last_known_addr { host: "127.22.50.124" port: 43653 } } }
20:03:56.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:56.920543 22977 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 7d8f4ab9d826436490a94ee04f174e7f [term 1 FOLLOWER]: Refusing update from remote peer 5c356dfb31e749c5a0f704baa1957a47: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
20:03:56.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:56.922297 23031 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 5c356dfb31e749c5a0f704baa1957a47 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7d8f4ab9d826436490a94ee04f174e7f" member_type: VOTER last_known_addr { host: "127.22.50.124" port: 43653 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
20:03:56.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:56.916064 22829 sys_catalog.cc:565] T 00000000000000000000000000000000 P 08b4dd9eba654c9e8faa59ad892a17ab [sys.catalog]: configured and running, proceeding with master startup.
20:03:56.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:56.925128 22802 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "5c356dfb31e749c5a0f704baa1957a47" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "08b4dd9eba654c9e8faa59ad892a17ab"
20:03:56.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:56.926009 22802 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 08b4dd9eba654c9e8faa59ad892a17ab [term 0 FOLLOWER]: Advancing to term 1
20:03:56.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:56.929965 22801 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "7d8f4ab9d826436490a94ee04f174e7f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "08b4dd9eba654c9e8faa59ad892a17ab" is_pre_election: true
20:03:56.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:56.918718 22804 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "7d8f4ab9d826436490a94ee04f174e7f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "08b4dd9eba654c9e8faa59ad892a17ab" is_pre_election: true
20:03:56.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:56.917088 22803 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "5c356dfb31e749c5a0f704baa1957a47" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "08b4dd9eba654c9e8faa59ad892a17ab" is_pre_election: true
20:03:56.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:56.935120 22950 leader_election.cc:304] T 00000000000000000000000000000000 P 7d8f4ab9d826436490a94ee04f174e7f [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 7d8f4ab9d826436490a94ee04f174e7f; no voters: 08b4dd9eba654c9e8faa59ad892a17ab, 5c356dfb31e749c5a0f704baa1957a47
20:03:56.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:56.932895 22804 rpcz_store.cc:267] Call kudu.consensus.ConsensusService.RequestConsensusVote from 127.0.0.1:59446 (request call id 2) took 1810 ms (client timeout 1710 ms). Trace:
20:03:56.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:56.935818 22804 rpcz_store.cc:269] 0503 20:03:55.122497 (+ 0us) service_pool.cc:167] Inserting onto call queue
20:03:56.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 20:03:55.122736 (+ 239us) service_pool.cc:224] Handling call
20:03:56.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 20:03:56.932809 (+1810073us) inbound_call.cc:177] Queueing success response
20:03:56.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
20:03:56.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:56.941220 22783 tablet.cc:2404] T 00000000000000000000000000000000 P 08b4dd9eba654c9e8faa59ad892a17ab: Can't schedule compaction. Clean time has not been advanced past its initial value.
20:03:56.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:56.914517 23050 sys_catalog.cc:458] T 00000000000000000000000000000000 P 08b4dd9eba654c9e8faa59ad892a17ab [sys.catalog]: This master's current role is: FOLLOWER
20:03:56.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:56.943113 22802 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 08b4dd9eba654c9e8faa59ad892a17ab [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 5c356dfb31e749c5a0f704baa1957a47 in term 1.
20:03:56.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:56.944293 22803 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 08b4dd9eba654c9e8faa59ad892a17ab [term 1 FOLLOWER]: Refusing update from remote peer 5c356dfb31e749c5a0f704baa1957a47: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
20:03:56.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:56.952090 23031 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 5c356dfb31e749c5a0f704baa1957a47 [LEADER]: Connected to new peer: Peer: permanent_uuid: "08b4dd9eba654c9e8faa59ad892a17ab" member_type: VOTER last_known_addr { host: "127.22.50.126" port: 34829 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
20:03:56.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:56.880146 22838 master_runner.cc:186] Error getting master registration for 127.22.50.126:34829: OK, instance_id { permanent_uuid: "08b4dd9eba654c9e8faa59ad892a17ab" instance_seqno: 1777838629770925 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
20:03:56.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:56.977543 22838 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:03:56.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:56.990324 23050 sys_catalog.cc:455] T 00000000000000000000000000000000 P 08b4dd9eba654c9e8faa59ad892a17ab [sys.catalog]: SysCatalogTable state changed. Reason: New leader 5c356dfb31e749c5a0f704baa1957a47. Latest consensus state: current_term: 1 leader_uuid: "5c356dfb31e749c5a0f704baa1957a47" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "08b4dd9eba654c9e8faa59ad892a17ab" member_type: VOTER last_known_addr { host: "127.22.50.126" port: 34829 } } peers { permanent_uuid: "5c356dfb31e749c5a0f704baa1957a47" member_type: VOTER last_known_addr { host: "127.22.50.125" port: 39091 } } peers { permanent_uuid: "7d8f4ab9d826436490a94ee04f174e7f" member_type: VOTER last_known_addr { host: "127.22.50.124" port: 43653 } } }
20:03:56.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:56.991071 23050 sys_catalog.cc:458] T 00000000000000000000000000000000 P 08b4dd9eba654c9e8faa59ad892a17ab [sys.catalog]: This master's current role is: FOLLOWER
20:03:56.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:56.997490 23051 mvcc.cc:204] Tried to move back new op lower bound from 7282027056802349056 to 7282027055888953344. Current Snapshot: MvccSnapshot[applied={T|T < 7282027056802349056}]
20:03:57.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:57.019327 23050 sys_catalog.cc:455] T 00000000000000000000000000000000 P 08b4dd9eba654c9e8faa59ad892a17ab [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "5c356dfb31e749c5a0f704baa1957a47" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "08b4dd9eba654c9e8faa59ad892a17ab" member_type: VOTER last_known_addr { host: "127.22.50.126" port: 34829 } } peers { permanent_uuid: "5c356dfb31e749c5a0f704baa1957a47" member_type: VOTER last_known_addr { host: "127.22.50.125" port: 39091 } } peers { permanent_uuid: "7d8f4ab9d826436490a94ee04f174e7f" member_type: VOTER last_known_addr { host: "127.22.50.124" port: 43653 } } }
20:03:57.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:57.020056 23050 sys_catalog.cc:458] T 00000000000000000000000000000000 P 08b4dd9eba654c9e8faa59ad892a17ab [sys.catalog]: This master's current role is: FOLLOWER
20:03:57.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:57.026861 23052 raft_consensus.cc:2749] T 00000000000000000000000000000000 P 7d8f4ab9d826436490a94ee04f174e7f [term 1 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
20:03:57.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:57.027851 23031 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5c356dfb31e749c5a0f704baa1957a47 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "5c356dfb31e749c5a0f704baa1957a47" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "08b4dd9eba654c9e8faa59ad892a17ab" member_type: VOTER last_known_addr { host: "127.22.50.126" port: 34829 } } peers { permanent_uuid: "5c356dfb31e749c5a0f704baa1957a47" member_type: VOTER last_known_addr { host: "127.22.50.125" port: 39091 } } peers { permanent_uuid: "7d8f4ab9d826436490a94ee04f174e7f" member_type: VOTER last_known_addr { host: "127.22.50.124" port: 43653 } } }
20:03:57.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:57.028568 23031 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5c356dfb31e749c5a0f704baa1957a47 [sys.catalog]: This master's current role is: LEADER
20:03:57.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:57.034032 23038 catalog_manager.cc:1357] Generated new cluster ID: 53f04c6e735c41bcaebd2b8fefbc6833
20:03:57.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:57.034391 23038 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
20:03:57.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:57.041953 23054 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7d8f4ab9d826436490a94ee04f174e7f [sys.catalog]: SysCatalogTable state changed. Reason: New leader 5c356dfb31e749c5a0f704baa1957a47. Latest consensus state: current_term: 1 leader_uuid: "5c356dfb31e749c5a0f704baa1957a47" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "08b4dd9eba654c9e8faa59ad892a17ab" member_type: VOTER last_known_addr { host: "127.22.50.126" port: 34829 } } peers { permanent_uuid: "5c356dfb31e749c5a0f704baa1957a47" member_type: VOTER last_known_addr { host: "127.22.50.125" port: 39091 } } peers { permanent_uuid: "7d8f4ab9d826436490a94ee04f174e7f" member_type: VOTER last_known_addr { host: "127.22.50.124" port: 43653 } } }
20:03:57.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:57.044284 23054 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7d8f4ab9d826436490a94ee04f174e7f [sys.catalog]: This master's current role is: FOLLOWER
20:03:57.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:57.061393 23032 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5c356dfb31e749c5a0f704baa1957a47 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "5c356dfb31e749c5a0f704baa1957a47" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "08b4dd9eba654c9e8faa59ad892a17ab" member_type: VOTER last_known_addr { host: "127.22.50.126" port: 34829 } } peers { permanent_uuid: "5c356dfb31e749c5a0f704baa1957a47" member_type: VOTER last_known_addr { host: "127.22.50.125" port: 39091 } } peers { permanent_uuid: "7d8f4ab9d826436490a94ee04f174e7f" member_type: VOTER last_known_addr { host: "127.22.50.124" port: 43653 } } }
20:03:57.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:57.062139 23032 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5c356dfb31e749c5a0f704baa1957a47 [sys.catalog]: This master's current role is: LEADER
20:03:57.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:57.064249 23059 mvcc.cc:204] Tried to move back new op lower bound from 7282027056802349056 to 7282027055888953344. Current Snapshot: MvccSnapshot[applied={T|T < 7282027056802349056}]
20:03:57.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:57.068159 23054 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7d8f4ab9d826436490a94ee04f174e7f [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "5c356dfb31e749c5a0f704baa1957a47" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "08b4dd9eba654c9e8faa59ad892a17ab" member_type: VOTER last_known_addr { host: "127.22.50.126" port: 34829 } } peers { permanent_uuid: "5c356dfb31e749c5a0f704baa1957a47" member_type: VOTER last_known_addr { host: "127.22.50.125" port: 39091 } } peers { permanent_uuid: "7d8f4ab9d826436490a94ee04f174e7f" member_type: VOTER last_known_addr { host: "127.22.50.124" port: 43653 } } }
20:03:57.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:57.069000 23054 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7d8f4ab9d826436490a94ee04f174e7f [sys.catalog]: This master's current role is: FOLLOWER
20:03:57.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:57.097513 23071 catalog_manager.cc:805] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
20:03:57.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:57.125615 23038 catalog_manager.cc:1380] Generated new certificate authority record
20:03:57.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:57.128111 23038 catalog_manager.cc:1514] Loading token signing keys...
20:03:57.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:57.170550 23038 catalog_manager.cc:6044] T 00000000000000000000000000000000 P 5c356dfb31e749c5a0f704baa1957a47: Generated new TSK 0
20:03:57.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:57.172039 23038 catalog_manager.cc:1524] Initializing in-progress tserver states...
20:03:57.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.22.50.65@KRBTEST.COM; defaulting to no policy
20:03:57.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:03:57.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.22.50.65@KRBTEST.COM" created.
20:03:57.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:03:57.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.22.50.65 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/krb5kdc/kudu.keytab.
20:03:57.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.22.50.65 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/krb5kdc/kudu.keytab.
20:03:57.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.22.50.65@KRBTEST.COM; defaulting to no policy
20:03:57.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:03:57.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.22.50.65@KRBTEST.COM" created.
20:03:57.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:03:57.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.22.50.65 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/krb5kdc/kudu.keytab.
20:03:57.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.22.50.65 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/krb5kdc/kudu.keytab.
20:03:57.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:57.277248 22729 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskqmogBp/build/tsan/bin/kudu
20:03:57.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskqmogBp/build/tsan/bin/kudu
20:03:57.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/ts-0/wal
20:03:57.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/ts-0/data
20:03:57.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:03:57.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:03:57.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:03:57.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:03:57.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:03:57.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:03:57.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/ts-0/logs
20:03:57.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/ts-0/data/info.pb
20:03:57.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:03:57.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:03:57.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:03:57.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:03:57.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:03:57.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:03:57.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:03:57.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:03:57.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
20:03:57.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:03:57.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.50.65:0
20:03:57.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.50.65
20:03:57.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.50.65
20:03:57.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:03:57.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.50.126:34829,127.22.50.125:39091,127.22.50.124:43653
20:03:57.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/krb5kdc/kudu.keytab
20:03:57.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.22.50.65
20:03:57.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:03:57.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:03:57.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:03:57.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
20:03:57.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.50.84:44541
20:03:57.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:03:57.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:03:57.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:03:57.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false with env {KRB5CCNAME=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:03:57.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:57.884140 23024 catalog_manager.cc:1269] Loaded cluster ID: 53f04c6e735c41bcaebd2b8fefbc6833
20:03:57.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:57.884799 23024 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 7d8f4ab9d826436490a94ee04f174e7f: loading cluster ID for follower catalog manager: success
20:03:57.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:57.893942 23024 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 7d8f4ab9d826436490a94ee04f174e7f: acquiring CA information for follower catalog manager: success
20:03:57.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:57.896225 23077 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:03:57.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:57.896974 23077 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:03:57.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:57.902118 23024 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 7d8f4ab9d826436490a94ee04f174e7f: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
20:03:57.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:57.901319 23077 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:03:57.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:57.961534 23077 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:03:57.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:57.963581 23077 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.22.50.65
20:03:58.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:58.039556 23077 tablet_server_runner.cc:78] Tablet server non-default flags:
20:03:58.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:03:58.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.50.84:44541
20:03:58.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:03:58.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:03:58.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/ts-0/data
20:03:58.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/ts-0/wal
20:03:58.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/krb5kdc/kudu.keytab
20:03:58.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.22.50.65
20:03:58.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:03:58.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:03:58.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.50.65:0
20:03:58.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:03:58.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:03:58.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:03:58.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:03:58.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:03:58.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:03:58.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/ts-0/data/info.pb
20:03:58.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.50.65
20:03:58.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:03:58.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.50.126:34829,127.22.50.125:39091,127.22.50.124:43653
20:03:58.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:03:58.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:03:58.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:03:58.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:03:58.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:03:58.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.50.65
20:03:58.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/ts-0/logs
20:03:58.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:03:58.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:03:58.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:03:58.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
20:03:58.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:03:58.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0ba7f54e15c9f673bf3e350f30b8d5f1a4037b79
20:03:58.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:03:58.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 03 May 2026 19:43:25 UTC on e7f111948823
20:03:58.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11731
20:03:58.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:03:58.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:58.041482 23077 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:03:58.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:58.043004 23077 file_cache.cc:492] Constructed file cache file cache with capacity 419430
20:03:58.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:58.067082 23083 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:03:58.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:58.126873 23071 catalog_manager.cc:1269] Loaded cluster ID: 53f04c6e735c41bcaebd2b8fefbc6833
20:03:58.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:58.127549 23071 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 08b4dd9eba654c9e8faa59ad892a17ab: loading cluster ID for follower catalog manager: success
20:03:58.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:58.164665 23071 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 08b4dd9eba654c9e8faa59ad892a17ab: acquiring CA information for follower catalog manager: success
20:03:58.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:58.209028 23071 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 08b4dd9eba654c9e8faa59ad892a17ab: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
20:03:58.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:58.574779 22233 debug-util.cc:398] Leaking SignalData structure 0x7b08000b6c80 after lost signal to thread 22163
20:03:58.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:58.575541 22233 debug-util.cc:398] Leaking SignalData structure 0x7b0800050960 after lost signal to thread 22236
20:03:59.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 20:03:59 dist-test-slave-b0rl krb5kdc[22741](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777838639, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.50.126@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:03:59.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:03:59.578106 22773 init.cc:295] Successfully reacquired a new kerberos TGT
20:04:00.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:59.463485 23082 debug-util.cc:398] Leaking SignalData structure 0x7b0800027120 after lost signal to thread 23077
20:04:00.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:04:00.991871 23082 kernel_stack_watchdog.cc:198] Thread 23077 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 399ms:
20:04:00.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
20:04:00.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
20:04:00.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
20:04:00.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
20:04:01.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:04:00.999223 23077 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 2.937s user 0.883s sys 1.674s
20:04:01.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:04:00.999634 23077 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 2.938s user 0.883s sys 1.674s
20:04:01.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:03:58.071694 23084 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:01.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:04:01.014446 23101 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:01.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:01.015332 23077 server_base.cc:1061] running on GCE node
20:04:01.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:01.017421 23077 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:04:01.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:01.029871 23077 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:04:01.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:01.031792 23077 hybrid_clock.cc:648] HybridClock initialized: now 1777838641031654 us; error 106 us; skew 500 ppm
20:04:01.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:01.033629 23077 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:04:01.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 20:04:01 dist-test-slave-b0rl krb5kdc[22741](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777838641, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.50.65@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:04:01.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:01.045104 23077 init.cc:377] Logged in from keytab as kudu/127.22.50.65@KRBTEST.COM (short username kudu)
20:04:01.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:01.094662 23077 webserver.cc:492] Webserver started at http://127.22.50.65:44161/ using document root <none> and password file <none>
20:04:01.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:01.096410 23077 fs_manager.cc:362] Metadata directory not provided
20:04:01.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:01.096956 23077 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:04:01.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:01.097716 23077 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
20:04:01.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:01.106421 23077 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/ts-0/data/instance:
20:04:01.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4479bf87dcc14a039b8d139297d6ec9f"
20:04:01.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-03 20:04:01 on dist-test-slave-b0rl"
20:04:01.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:01.108634 23077 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/ts-0/wal/instance:
20:04:01.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4479bf87dcc14a039b8d139297d6ec9f"
20:04:01.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-03 20:04:01 on dist-test-slave-b0rl"
20:04:01.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:01.137208 23077 fs_manager.cc:696] Time spent creating directory manager: real 0.027s user 0.016s sys 0.009s
20:04:01.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:01.152179 23107 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:04:01.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:01.153688 23077 fs_manager.cc:730] Time spent opening block manager: real 0.011s user 0.007s sys 0.005s
20:04:01.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:01.154130 23077 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/ts-0/data,/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/ts-0/wal
20:04:01.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4479bf87dcc14a039b8d139297d6ec9f"
20:04:01.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-03 20:04:01 on dist-test-slave-b0rl"
20:04:01.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:01.154625 23077 fs_report.cc:389] FS layout report
20:04:01.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:04:01.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/ts-0/wal
20:04:01.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/ts-0/wal
20:04:01.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/ts-0/data/data
20:04:01.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:04:01.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:04:01.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:04:01.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:04:01.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:04:01.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:04:01.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:04:01.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:04:01.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:04:01.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:04:01.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:04:01.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:01.250669 23077 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:04:01.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:01.260388 23077 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:04:01.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:01.261103 23077 kserver.cc:163] Server-wide thread pool size limit: 3276
20:04:01.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:01.271078 23077 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
20:04:01.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:01.283601 23077 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
20:04:01.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:01.283970 23077 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
20:04:01.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:01.284268 23077 ts_tablet_manager.cc:616] Registered 0 tablets
20:04:01.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:01.284555 23077 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
20:04:01.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:04:01.392604 22861 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.22.50.126:34829, user_credentials={real_user=slave}} blocked reactor thread for 43617us
20:04:01.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 20:04:01 dist-test-slave-b0rl krb5kdc[22741](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777838641, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.50.125@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:04:01.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:01.771345 22855 init.cc:295] Successfully reacquired a new kerberos TGT
20:04:01.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:01.879890 23077 rpc_server.cc:307] RPC server started. Bound to: 127.22.50.65:37573
20:04:01.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:01.880055 23223 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.22.50.65:37573 every 8 connection(s)
20:04:01.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:01.883641 23077 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/ts-0/data/info.pb
20:04:01.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:01.885164 22729 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskqmogBp/build/tsan/bin/kudu as pid 23077
20:04:01.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:01.885664 22729 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/ts-0/wal/instance
20:04:01.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 20:04:01 dist-test-slave-b0rl krb5kdc[22741](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777838641, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.50.65@KRBTEST.COM for kudu/127.22.50.124@KRBTEST.COM
20:04:02.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.22.50.66@KRBTEST.COM; defaulting to no policy
20:04:02.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:04:02.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.22.50.66@KRBTEST.COM" created.
20:04:02.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:02.042227 23224 heartbeater.cc:344] Connected to a master server at 127.22.50.124:43653
20:04:02.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:02.042943 23224 heartbeater.cc:461] Registering TS with master...
20:04:02.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:02.044417 23224 heartbeater.cc:507] Master 127.22.50.124:43653 requested a full tablet report, sending...
20:04:02.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 20:04:02 dist-test-slave-b0rl krb5kdc[22741](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777838641, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.50.65@KRBTEST.COM for kudu/127.22.50.125@KRBTEST.COM
20:04:02.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:02.049350 22967 ts_manager.cc:194] Registered new tserver with Master: 4479bf87dcc14a039b8d139297d6ec9f (127.22.50.65:37573)
20:04:02.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 20:04:02 dist-test-slave-b0rl krb5kdc[22741](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777838641, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.50.65@KRBTEST.COM for kudu/127.22.50.126@KRBTEST.COM
20:04:02.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:02.075631 23225 heartbeater.cc:344] Connected to a master server at 127.22.50.125:39091
20:04:02.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:02.076020 23225 heartbeater.cc:461] Registering TS with master...
20:04:02.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:02.077011 23225 heartbeater.cc:507] Master 127.22.50.125:39091 requested a full tablet report, sending...
20:04:02.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:02.079877 22876 ts_manager.cc:194] Registered new tserver with Master: 4479bf87dcc14a039b8d139297d6ec9f (127.22.50.65:37573)
20:04:02.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:02.085794 22876 master_service.cc:502] Signed X509 certificate for tserver {username='kudu', principal='kudu/127.22.50.65@KRBTEST.COM'} at 127.22.50.65:49759
20:04:02.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:02.101239 23229 heartbeater.cc:344] Connected to a master server at 127.22.50.126:34829
20:04:02.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:02.101692 23229 heartbeater.cc:461] Registering TS with master...
20:04:02.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:02.103045 23229 heartbeater.cc:507] Master 127.22.50.126:34829 requested a full tablet report, sending...
20:04:02.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:04:02.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.22.50.66 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/krb5kdc/kudu.keytab.
20:04:02.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.22.50.66 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/krb5kdc/kudu.keytab.
20:04:02.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:02.106992 22794 ts_manager.cc:194] Registered new tserver with Master: 4479bf87dcc14a039b8d139297d6ec9f (127.22.50.65:37573)
20:04:02.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:04:02.108428 22729 mini_kdc.cc:268] Time spent creating service keytab for kudu/127.22.50.66: real 0.213s user 0.000s sys 0.014s
20:04:02.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.22.50.66@KRBTEST.COM; defaulting to no policy
20:04:02.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:04:02.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.22.50.66@KRBTEST.COM" created.
20:04:02.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:04:02.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.22.50.66 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/krb5kdc/kudu.keytab.
20:04:02.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.22.50.66 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/krb5kdc/kudu.keytab.
20:04:02.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:02.166193 22729 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskqmogBp/build/tsan/bin/kudu
20:04:02.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskqmogBp/build/tsan/bin/kudu
20:04:02.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/ts-1/wal
20:04:02.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/ts-1/data
20:04:02.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:04:02.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:04:02.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:04:02.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:04:02.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:04:02.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:04:02.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/ts-1/logs
20:04:02.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/ts-1/data/info.pb
20:04:02.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:04:02.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:04:02.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:04:02.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:04:02.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:04:02.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:04:02.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:04:02.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:04:02.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
20:04:02.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:04:02.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.50.66:0
20:04:02.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.50.66
20:04:02.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.50.66
20:04:02.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:04:02.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.50.126:34829,127.22.50.125:39091,127.22.50.124:43653
20:04:02.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/krb5kdc/kudu.keytab
20:04:02.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.22.50.66
20:04:02.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:04:02.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:04:02.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:04:02.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
20:04:02.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.50.84:44541
20:04:02.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:04:02.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:04:02.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:04:02.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false with env {KRB5CCNAME=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:04:02.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:04:02.629571 23238 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:04:02.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:04:02.630211 23238 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:04:02.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:04:02.631218 23238 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:04:02.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:04:02.685014 23238 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:04:02.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:04:02.686129 23238 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.22.50.66
20:04:02.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:02.732985 23238 tablet_server_runner.cc:78] Tablet server non-default flags:
20:04:02.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:04:02.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.50.84:44541
20:04:02.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:04:02.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:04:02.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/ts-1/data
20:04:02.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/ts-1/wal
20:04:02.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/krb5kdc/kudu.keytab
20:04:02.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.22.50.66
20:04:02.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:04:02.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:04:02.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.50.66:0
20:04:02.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:04:02.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:04:02.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:04:02.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:04:02.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:04:02.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:04:02.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/ts-1/data/info.pb
20:04:02.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.50.66
20:04:02.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:04:02.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.50.126:34829,127.22.50.125:39091,127.22.50.124:43653
20:04:02.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:04:02.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:04:02.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:04:02.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:04:02.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:04:02.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.50.66
20:04:02.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/ts-1/logs
20:04:02.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:04:02.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:04:02.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:04:02.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
20:04:02.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:04:02.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0ba7f54e15c9f673bf3e350f30b8d5f1a4037b79
20:04:02.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:04:02.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 03 May 2026 19:43:25 UTC on e7f111948823
20:04:02.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11731
20:04:02.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:04:02.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:02.735870 23238 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:04:02.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:02.737257 23238 file_cache.cc:492] Constructed file cache file cache with capacity 419430
20:04:02.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:04:02.756628 23247 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:03.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:03.090340 23225 heartbeater.cc:499] Master 127.22.50.125:39091 was elected leader, sending a full tablet report...
20:04:04.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:04:04.158874 23246 debug-util.cc:398] Leaking SignalData structure 0x7b0800027120 after lost signal to thread 23238
20:04:04.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:04:02.760452 23248 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:04.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:04:04.196853 23238 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 1.438s user 0.553s sys 0.853s
20:04:04.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:04:04.197250 23238 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 1.439s user 0.553s sys 0.853s
20:04:04.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:04:04.209074 23251 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:04.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:04.209105 23238 server_base.cc:1061] running on GCE node
20:04:04.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:04.210553 23238 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:04:04.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:04.212831 23238 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:04:04.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:04.214277 23238 hybrid_clock.cc:648] HybridClock initialized: now 1777838644214109 us; error 168 us; skew 500 ppm
20:04:04.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:04.215302 23238 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:04:04.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 20:04:04 dist-test-slave-b0rl krb5kdc[22741](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777838644, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.50.66@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:04:04.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:04.223227 23238 init.cc:377] Logged in from keytab as kudu/127.22.50.66@KRBTEST.COM (short username kudu)
20:04:04.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:04.229506 23238 webserver.cc:492] Webserver started at http://127.22.50.66:34923/ using document root <none> and password file <none>
20:04:04.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:04.230860 23238 fs_manager.cc:362] Metadata directory not provided
20:04:04.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:04.231185 23238 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:04:04.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:04.231727 23238 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
20:04:04.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:04.236227 23238 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/ts-1/data/instance:
20:04:04.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d4d95fecf2ad4ff88f496160b149a076"
20:04:04.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-03 20:04:04 on dist-test-slave-b0rl"
20:04:04.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:04.238267 23238 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/ts-1/wal/instance:
20:04:04.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d4d95fecf2ad4ff88f496160b149a076"
20:04:04.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-03 20:04:04 on dist-test-slave-b0rl"
20:04:04.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:04.248013 23238 fs_manager.cc:696] Time spent creating directory manager: real 0.009s user 0.006s sys 0.002s
20:04:04.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:04.255378 23258 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:04:04.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:04.256610 23238 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.001s sys 0.003s
20:04:04.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:04.257056 23238 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/ts-1/data,/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/ts-1/wal
20:04:04.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d4d95fecf2ad4ff88f496160b149a076"
20:04:04.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-03 20:04:04 on dist-test-slave-b0rl"
20:04:04.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:04.257433 23238 fs_report.cc:389] FS layout report
20:04:04.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:04:04.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/ts-1/wal
20:04:04.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/ts-1/wal
20:04:04.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/ts-1/data/data
20:04:04.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:04:04.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:04:04.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:04:04.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:04:04.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:04:04.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:04:04.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:04:04.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:04:04.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:04:04.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:04:04.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:04:04.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 20:04:04 dist-test-slave-b0rl krb5kdc[22741](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777838644, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.50.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:04:04.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:04.310168 22944 init.cc:295] Successfully reacquired a new kerberos TGT
20:04:04.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:04.317521 23238 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:04:04.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:04.324028 23238 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:04:04.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:04.324628 23238 kserver.cc:163] Server-wide thread pool size limit: 3276
20:04:04.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:04.327503 23238 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
20:04:04.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:04.332881 23238 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
20:04:04.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:04.333097 23238 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
20:04:04.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:04.333302 23238 ts_tablet_manager.cc:616] Registered 0 tablets
20:04:04.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:04.333439 23238 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
20:04:04.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:04.580859 23238 rpc_server.cc:307] RPC server started. Bound to: 127.22.50.66:32791
20:04:04.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:04.580936 23370 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.22.50.66:32791 every 8 connection(s)
20:04:04.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:04.584748 23238 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/ts-1/data/info.pb
20:04:04.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:04.593890 22729 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskqmogBp/build/tsan/bin/kudu as pid 23238
20:04:04.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:04.594339 22729 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/ts-1/wal/instance
20:04:04.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 20:04:04 dist-test-slave-b0rl krb5kdc[22741](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777838644, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.50.66@KRBTEST.COM for kudu/127.22.50.125@KRBTEST.COM
20:04:04.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.22.50.67@KRBTEST.COM; defaulting to no policy
20:04:04.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:04:04.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.22.50.67@KRBTEST.COM" created.
20:04:04.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 20:04:04 dist-test-slave-b0rl krb5kdc[22741](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777838644, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.50.66@KRBTEST.COM for kudu/127.22.50.124@KRBTEST.COM
20:04:04.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 20:04:04 dist-test-slave-b0rl krb5kdc[22741](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777838644, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.50.66@KRBTEST.COM for kudu/127.22.50.126@KRBTEST.COM
20:04:04.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:04.678918 23372 heartbeater.cc:344] Connected to a master server at 127.22.50.125:39091
20:04:04.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:04.679539 23372 heartbeater.cc:461] Registering TS with master...
20:04:04.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:04.681627 23372 heartbeater.cc:507] Master 127.22.50.125:39091 requested a full tablet report, sending...
20:04:04.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:04.684490 22876 ts_manager.cc:194] Registered new tserver with Master: d4d95fecf2ad4ff88f496160b149a076 (127.22.50.66:32791)
20:04:04.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:04.686327 23371 heartbeater.cc:344] Connected to a master server at 127.22.50.124:43653
20:04:04.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:04.686754 23371 heartbeater.cc:461] Registering TS with master...
20:04:04.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:04.686363 22876 master_service.cc:502] Signed X509 certificate for tserver {username='kudu', principal='kudu/127.22.50.66@KRBTEST.COM'} at 127.22.50.66:43109
20:04:04.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:04.687654 23373 heartbeater.cc:344] Connected to a master server at 127.22.50.126:34829
20:04:04.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:04.688066 23373 heartbeater.cc:461] Registering TS with master...
20:04:04.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:04.687943 23371 heartbeater.cc:507] Master 127.22.50.124:43653 requested a full tablet report, sending...
20:04:04.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:04.689327 23373 heartbeater.cc:507] Master 127.22.50.126:34829 requested a full tablet report, sending...
20:04:04.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:04.691752 22967 ts_manager.cc:194] Registered new tserver with Master: d4d95fecf2ad4ff88f496160b149a076 (127.22.50.66:32791)
20:04:04.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:04.692493 22794 ts_manager.cc:194] Registered new tserver with Master: d4d95fecf2ad4ff88f496160b149a076 (127.22.50.66:32791)
20:04:04.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:04:04.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.22.50.67 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/krb5kdc/kudu.keytab.
20:04:04.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.22.50.67 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/krb5kdc/kudu.keytab.
20:04:04.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:04:04.711999 22729 mini_kdc.cc:268] Time spent creating service keytab for kudu/127.22.50.67: real 0.101s user 0.004s sys 0.010s
20:04:04.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.22.50.67@KRBTEST.COM; defaulting to no policy
20:04:04.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:04:04.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.22.50.67@KRBTEST.COM" created.
20:04:04.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:04:04.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.22.50.67 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/krb5kdc/kudu.keytab.
20:04:04.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.22.50.67 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/krb5kdc/kudu.keytab.
20:04:04.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:04.760810 22729 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskqmogBp/build/tsan/bin/kudu
20:04:04.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskqmogBp/build/tsan/bin/kudu
20:04:04.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/ts-2/wal
20:04:04.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/ts-2/data
20:04:04.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:04:04.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:04:04.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:04:04.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:04:04.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:04:04.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:04:04.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/ts-2/logs
20:04:04.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/ts-2/data/info.pb
20:04:04.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:04:04.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:04:04.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:04:04.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:04:04.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:04:04.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:04:04.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:04:04.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:04:04.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
20:04:04.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:04:04.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.50.67:0
20:04:04.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.50.67
20:04:04.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.50.67
20:04:04.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:04:04.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.50.126:34829,127.22.50.125:39091,127.22.50.124:43653
20:04:04.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/krb5kdc/kudu.keytab
20:04:04.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.22.50.67
20:04:04.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:04:04.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:04:04.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:04:04.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
20:04:04.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.50.84:44541
20:04:04.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:04:04.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:04:04.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:04:04.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false with env {KRB5CCNAME=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:04:05.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:04:05.116639 23385 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:04:05.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:04:05.117115 23385 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:04:05.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:04:05.117736 23385 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:04:05.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:04:05.154049 23385 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:04:05.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:04:05.155035 23385 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.22.50.67
20:04:05.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:05.198659 23385 tablet_server_runner.cc:78] Tablet server non-default flags:
20:04:05.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:04:05.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.50.84:44541
20:04:05.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:04:05.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:04:05.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/ts-2/data
20:04:05.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/ts-2/wal
20:04:05.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/krb5kdc/kudu.keytab
20:04:05.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.22.50.67
20:04:05.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:04:05.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:04:05.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.50.67:0
20:04:05.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:04:05.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:04:05.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:04:05.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:04:05.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:04:05.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:04:05.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/ts-2/data/info.pb
20:04:05.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.50.67
20:04:05.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:04:05.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.50.126:34829,127.22.50.125:39091,127.22.50.124:43653
20:04:05.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:04:05.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:04:05.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:04:05.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:04:05.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:04:05.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.50.67
20:04:05.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/ts-2/logs
20:04:05.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:04:05.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:04:05.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:04:05.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
20:04:05.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:04:05.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0ba7f54e15c9f673bf3e350f30b8d5f1a4037b79
20:04:05.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:04:05.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 03 May 2026 19:43:25 UTC on e7f111948823
20:04:05.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11731
20:04:05.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:04:05.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:05.199949 23385 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:04:05.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:05.200971 23385 file_cache.cc:492] Constructed file cache file cache with capacity 419430
20:04:05.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:04:05.215507 23391 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:05.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:04:05.517202 22825 debug-util.cc:398] Leaking SignalData structure 0x7b08000925c0 after lost signal to thread 22759
20:04:05.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:05.692243 23372 heartbeater.cc:499] Master 127.22.50.125:39091 was elected leader, sending a full tablet report...
20:04:07.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:04:06.616978 23390 debug-util.cc:398] Leaking SignalData structure 0x7b0800027120 after lost signal to thread 23385
20:04:07.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:04:07.560672 23394 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:07.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:04:07.560871 23393 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2343 milliseconds
20:04:07.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:04:05.216293 23392 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:07.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:04:07.559765 23385 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 2.345s user 0.569s sys 1.039s
20:04:07.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:04:07.561616 23385 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 2.347s user 0.569s sys 1.039s
20:04:07.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:07.561928 23385 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:04:07.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:07.566212 23385 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:04:07.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:07.568602 23385 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:04:07.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:07.569980 23385 hybrid_clock.cc:648] HybridClock initialized: now 1777838647569931 us; error 66 us; skew 500 ppm
20:04:07.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:07.570972 23385 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:04:07.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 20:04:07 dist-test-slave-b0rl krb5kdc[22741](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777838647, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.50.67@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:04:07.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:07.580060 23385 init.cc:377] Logged in from keytab as kudu/127.22.50.67@KRBTEST.COM (short username kudu)
20:04:07.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:07.605875 23385 webserver.cc:492] Webserver started at http://127.22.50.67:41307/ using document root <none> and password file <none>
20:04:07.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:07.607192 23385 fs_manager.cc:362] Metadata directory not provided
20:04:07.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:07.607483 23385 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:04:07.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:07.608047 23385 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
20:04:07.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:07.614542 23385 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/ts-2/data/instance:
20:04:07.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a84386050e7f4ddf8d0c6b6f64c59135"
20:04:07.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-03 20:04:07 on dist-test-slave-b0rl"
20:04:07.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:07.616106 23385 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/ts-2/wal/instance:
20:04:07.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a84386050e7f4ddf8d0c6b6f64c59135"
20:04:07.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-03 20:04:07 on dist-test-slave-b0rl"
20:04:07.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:07.626657 23385 fs_manager.cc:696] Time spent creating directory manager: real 0.010s user 0.005s sys 0.003s
20:04:07.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:07.634688 23406 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:04:07.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:07.636205 23385 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.003s sys 0.000s
20:04:07.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:07.636615 23385 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/ts-2/data,/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/ts-2/wal
20:04:07.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a84386050e7f4ddf8d0c6b6f64c59135"
20:04:07.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-03 20:04:07 on dist-test-slave-b0rl"
20:04:07.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:07.637125 23385 fs_report.cc:389] FS layout report
20:04:07.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:04:07.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/ts-2/wal
20:04:07.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/ts-2/wal
20:04:07.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/ts-2/data/data
20:04:07.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:04:07.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:04:07.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:04:07.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:04:07.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:04:07.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:04:07.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:04:07.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:04:07.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:04:07.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:04:07.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:04:07.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:04:07.657740 22571 debug-util.cc:398] Leaking SignalData structure 0x7b08000ade80 after lost signal to thread 22446
20:04:07.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:04:07.659113 22571 debug-util.cc:398] Leaking SignalData structure 0x7b08000ba040 after lost signal to thread 22574
20:04:07.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:07.714737 23385 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:04:07.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:07.738451 23385 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:04:07.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:07.739147 23385 kserver.cc:163] Server-wide thread pool size limit: 3276
20:04:07.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:07.743607 23385 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
20:04:07.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:07.750428 23385 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
20:04:07.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:07.750803 23385 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.001s sys 0.000s
20:04:07.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:07.751134 23385 ts_tablet_manager.cc:616] Registered 0 tablets
20:04:07.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:07.751364 23385 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
20:04:08.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:08.163303 23385 rpc_server.cc:307] RPC server started. Bound to: 127.22.50.67:35917
20:04:08.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:08.163446 23519 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.22.50.67:35917 every 8 connection(s)
20:04:08.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:08.167446 23385 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/ts-2/data/info.pb
20:04:08.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:08.174384 22729 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskqmogBp/build/tsan/bin/kudu as pid 23385
20:04:08.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:08.175441 22729 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/ts-2/wal/instance
20:04:08.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 20:04:08 dist-test-slave-b0rl krb5kdc[22741](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777838647, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.50.67@KRBTEST.COM for kudu/127.22.50.125@KRBTEST.COM
20:04:08.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 20:04:08 dist-test-slave-b0rl krb5kdc[22741](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777838647, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.50.67@KRBTEST.COM for kudu/127.22.50.124@KRBTEST.COM
20:04:08.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 20:04:08 dist-test-slave-b0rl krb5kdc[22741](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777838647, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.50.67@KRBTEST.COM for kudu/127.22.50.126@KRBTEST.COM
20:04:08.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:08.391613 23521 heartbeater.cc:344] Connected to a master server at 127.22.50.125:39091
20:04:08.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:08.392141 23521 heartbeater.cc:461] Registering TS with master...
20:04:08.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:08.393673 23521 heartbeater.cc:507] Master 127.22.50.125:39091 requested a full tablet report, sending...
20:04:08.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:08.398638 22876 ts_manager.cc:194] Registered new tserver with Master: a84386050e7f4ddf8d0c6b6f64c59135 (127.22.50.67:35917)
20:04:08.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:08.400926 22876 master_service.cc:502] Signed X509 certificate for tserver {username='kudu', principal='kudu/127.22.50.67@KRBTEST.COM'} at 127.22.50.67:43311
20:04:08.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:08.404421 23520 heartbeater.cc:344] Connected to a master server at 127.22.50.124:43653
20:04:08.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:08.404834 23520 heartbeater.cc:461] Registering TS with master...
20:04:08.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:08.405966 23520 heartbeater.cc:507] Master 127.22.50.124:43653 requested a full tablet report, sending...
20:04:08.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:08.408396 23525 heartbeater.cc:344] Connected to a master server at 127.22.50.126:34829
20:04:08.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:08.408771 23525 heartbeater.cc:461] Registering TS with master...
20:04:08.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:08.409550 22967 ts_manager.cc:194] Registered new tserver with Master: a84386050e7f4ddf8d0c6b6f64c59135 (127.22.50.67:35917)
20:04:08.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:08.409740 23525 heartbeater.cc:507] Master 127.22.50.126:34829 requested a full tablet report, sending...
20:04:08.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:08.413504 22794 ts_manager.cc:194] Registered new tserver with Master: a84386050e7f4ddf8d0c6b6f64c59135 (127.22.50.67:35917)
20:04:08.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:08.439898 22729 external_mini_cluster.cc:949] 3 TS(s) registered with all masters
20:04:08.454 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:04:08.455 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
20:04:08.465 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
20:04:08.467 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
20:04:08.472 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.22.50.126"
port: 34829
}
bound_http_address {
host: "127.22.50.126"
port: 40523
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.22.50.125"
port: 39091
}
bound_http_address {
host: "127.22.50.125"
port: 40527
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.22.50.124"
port: 43653
}
bound_http_address {
host: "127.22.50.124"
port: 36701
}
}
}
20:04:08.472 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
20:04:08.474 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.22.50.65"
port: 37573
}
bound_http_address {
host: "127.22.50.65"
port: 44161
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.22.50.66"
port: 32791
}
bound_http_address {
host: "127.22.50.66"
port: 34923
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.22.50.67"
port: 35917
}
bound_http_address {
host: "127.22.50.67"
port: 41307
}
}
}
20:04:08.474 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
20:04:08.475 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
20:04:08.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 20:04:08 dist-test-slave-b0rl krb5kdc[22741](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777838648, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:04:08.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
20:04:08.536 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:04:08.538 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/krb5kdc/krb5cc
20:04:08.548 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
20:04:08.549 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/krb5kdc/krb5cc
20:04:08.551 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
20:04:08.581 [DEBUG - Time-limited test] (SecurityContext.java:158) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
20:04:08.587 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
20:04:08.587 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
20:04:08.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 20:04:08 dist-test-slave-b0rl krb5kdc[22741](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777838648, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:04:08.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
20:04:08.649 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:04:08.649 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/krb5kdc/krb5cc
20:04:08.651 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
20:04:08.652 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.22.50.125:39091
20:04:08.654 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
20:04:08.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:08.654867 22729 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskqmogBp/build/tsan/bin/kudu with pid 22838
20:04:08.700 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:04:08.701 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.22.50.124:43653
20:04:08.701 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
20:04:08.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:08.701737 22729 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskqmogBp/build/tsan/bin/kudu with pid 22921
20:04:08.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:04:08.741986 23261 connection.cc:570] client connection to 127.22.50.124:43653 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
20:04:08.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:04:08.742810 23371 heartbeater.cc:646] Failed to heartbeat to 127.22.50.124:43653 (0 consecutive failures): Network error: Failed to send heartbeat to master: recv error from unknown peer: Transport endpoint is not connected (error 107)
20:04:08.744 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:04:08.744 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.22.50.126:34829
20:04:08.745 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
20:04:08.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:08.745654 22729 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskqmogBp/build/tsan/bin/kudu with pid 22758
20:04:08.779 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:04:08.780 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.22.50.125:39091
20:04:08.786 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
20:04:08.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:08.788179 22729 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskqmogBp/build/tsan/bin/kudu
20:04:08.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskqmogBp/build/tsan/bin/kudu
20:04:08.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/master-1/wal
20:04:08.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/master-1/data
20:04:08.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:04:08.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:04:08.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:04:08.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:04:08.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:04:08.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:04:08.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/master-1/logs
20:04:08.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/master-1/data/info.pb
20:04:08.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:04:08.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:04:08.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:04:08.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:04:08.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:04:08.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:04:08.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:04:08.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:04:08.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:04:08.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:04:08.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:04:08.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:04:08.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.50.125:39091
20:04:08.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.50.125
20:04:08.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=40527
20:04:08.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/krb5kdc/kudu.keytab
20:04:08.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.22.50.125
20:04:08.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:04:08.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:04:08.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:04:08.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
20:04:08.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.50.84:44541
20:04:08.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:04:08.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:04:08.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:04:08.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:04:08.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.50.126:34829,127.22.50.125:39091,127.22.50.124:43653
20:04:08.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:04:08.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:32783/jwks.json
20:04:08.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
20:04:08.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:04:09.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:04:09.088848 23224 heartbeater.cc:646] Failed to heartbeat to 127.22.50.124:43653 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.22.50.124:43653: connect: Connection refused (error 111)
20:04:09.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:04:09.142927 23538 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:04:09.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:04:09.143641 23538 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:04:09.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:04:09.144598 23538 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:04:09.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:04:09.189178 23538 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
20:04:09.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:04:09.189429 23538 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
20:04:09.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:04:09.189704 23538 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:04:09.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:04:09.189903 23538 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
20:04:09.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:04:09.190101 23538 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
20:04:09.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:04:09.190290 23538 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
20:04:09.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:04:09.190474 23538 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:32783/jwks.json
20:04:09.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:09.231827 23538 master_runner.cc:386] Master server non-default flags:
20:04:09.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:04:09.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.50.84:44541
20:04:09.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:04:09.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:04:09.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/master-1/data
20:04:09.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/master-1/wal
20:04:09.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
20:04:09.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:04:09.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.50.126:34829,127.22.50.125:39091,127.22.50.124:43653
20:04:09.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/krb5kdc/kudu.keytab
20:04:09.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.22.50.125
20:04:09.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:04:09.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:04:09.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:04:09.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.50.125:39091
20:04:09.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:04:09.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:04:09.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:04:09.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:32783/jwks.json
20:04:09.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:04:09.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:04:09.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:04:09.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:04:09.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:04:09.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/master-1/data/info.pb
20:04:09.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.50.125
20:04:09.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=40527
20:04:09.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:04:09.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:04:09.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:04:09.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:04:09.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:04:09.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/master-1/logs
20:04:09.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:04:09.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:04:09.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:04:09.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:04:09.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:04:09.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0ba7f54e15c9f673bf3e350f30b8d5f1a4037b79
20:04:09.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:04:09.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 03 May 2026 19:43:25 UTC on e7f111948823
20:04:09.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11731
20:04:09.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:04:09.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:09.233357 23538 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:04:09.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:09.234905 23538 file_cache.cc:492] Constructed file cache file cache with capacity 419430
20:04:09.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:04:09.248448 23546 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:09.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:09.414242 23521 heartbeater.cc:499] Master 127.22.50.125:39091 was elected leader, sending a full tablet report...
20:04:09.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:04:09.425670 23521 heartbeater.cc:646] Failed to heartbeat to 127.22.50.125:39091 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.22.50.125:39091: connect: Connection refused (error 111)
20:04:11.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 20:04:11 dist-test-slave-b0rl krb5kdc[22741](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777838651, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.50.65@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:04:11.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:11.072366 23103 init.cc:295] Successfully reacquired a new kerberos TGT
20:04:11.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:04:10.650630 23545 debug-util.cc:398] Leaking SignalData structure 0x7b0800028d00 after lost signal to thread 23538
20:04:11.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:04:11.397425 23545 kernel_stack_watchdog.cc:198] Thread 23538 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 398ms:
20:04:11.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
20:04:11.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
20:04:11.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
20:04:11.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
20:04:11.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:04:09.249085 23547 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:11.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:04:11.398546 23538 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 2.148s user 0.683s sys 1.159s
20:04:11.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:04:11.399012 23538 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 2.151s user 0.684s sys 1.159s
20:04:11.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:04:11.403332 23548 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2148 milliseconds
20:04:11.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:04:11.405211 23550 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:11.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:11.405789 23538 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:04:11.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:11.407117 23538 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:04:11.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:11.412755 23538 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:04:11.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:11.414669 23538 hybrid_clock.cc:648] HybridClock initialized: now 1777838651414242 us; error 218 us; skew 500 ppm
20:04:11.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:11.416988 23538 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:04:11.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 20:04:11 dist-test-slave-b0rl krb5kdc[22741](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777838651, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.50.125@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:04:11.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:11.425019 23538 init.cc:377] Logged in from keytab as kudu/127.22.50.125@KRBTEST.COM (short username kudu)
20:04:11.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:11.435403 23538 webserver.cc:492] Webserver started at http://127.22.50.125:40527/ using document root <none> and password file <none>
20:04:11.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:11.436517 23538 fs_manager.cc:362] Metadata directory not provided
20:04:11.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:11.437095 23538 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:04:11.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:11.446430 23538 fs_manager.cc:714] Time spent opening directory manager: real 0.006s user 0.002s sys 0.004s
20:04:11.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:11.452970 23574 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:04:11.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:11.454279 23538 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.003s sys 0.003s
20:04:11.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:11.454682 23538 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/master-1/data,/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/master-1/wal
20:04:11.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5c356dfb31e749c5a0f704baa1957a47"
20:04:11.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-03 20:03:51 on dist-test-slave-b0rl"
20:04:11.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:11.457177 23538 fs_report.cc:389] FS layout report
20:04:11.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:04:11.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/master-1/wal
20:04:11.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/master-1/wal
20:04:11.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/master-1/data/data
20:04:11.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:04:11.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:04:11.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:04:11.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:04:11.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:04:11.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:04:11.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:04:11.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:04:11.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:04:11.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:04:11.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:04:11.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:11.529743 23538 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:04:11.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:11.535600 23538 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:04:11.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:11.536036 23538 kserver.cc:163] Server-wide thread pool size limit: 3276
20:04:11.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:11.619458 23538 rpc_server.cc:307] RPC server started. Bound to: 127.22.50.125:39091
20:04:11.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:11.619624 23625 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.22.50.125:39091 every 8 connection(s)
20:04:11.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:11.622345 23538 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/master-1/data/info.pb
20:04:11.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:11.625231 22729 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskqmogBp/build/tsan/bin/kudu as pid 23538
20:04:11.626 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:04:11.626 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.22.50.124:43653
20:04:11.626 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
20:04:11.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:11.627753 22729 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskqmogBp/build/tsan/bin/kudu
20:04:11.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskqmogBp/build/tsan/bin/kudu
20:04:11.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/master-2/wal
20:04:11.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/master-2/data
20:04:11.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:04:11.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:04:11.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:04:11.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:04:11.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:04:11.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:04:11.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/master-2/logs
20:04:11.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/master-2/data/info.pb
20:04:11.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:04:11.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:04:11.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:04:11.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:04:11.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:04:11.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:04:11.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:04:11.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:04:11.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:04:11.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:04:11.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:04:11.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:04:11.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.50.124:43653
20:04:11.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.50.124
20:04:11.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36701
20:04:11.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/krb5kdc/kudu.keytab
20:04:11.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.22.50.124
20:04:11.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:04:11.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:04:11.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:04:11.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
20:04:11.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.50.84:44541
20:04:11.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:04:11.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:04:11.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:04:11.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:04:11.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.50.126:34829,127.22.50.125:39091,127.22.50.124:43653
20:04:11.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:04:11.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:32783/jwks.json
20:04:11.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
20:04:11.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:04:11.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:11.638872 23626 sys_catalog.cc:263] Verifying existing consensus state
20:04:11.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:11.647418 23626 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5c356dfb31e749c5a0f704baa1957a47: Bootstrap starting.
20:04:11.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:11.693455 23626 log.cc:826] T 00000000000000000000000000000000 P 5c356dfb31e749c5a0f704baa1957a47: Log is configured to *not* fsync() on all Append() calls
20:04:11.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:11.709851 23626 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5c356dfb31e749c5a0f704baa1957a47: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
20:04:11.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:11.710927 23626 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5c356dfb31e749c5a0f704baa1957a47: Bootstrap complete.
20:04:11.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:11.735340 23626 raft_consensus.cc:359] T 00000000000000000000000000000000 P 5c356dfb31e749c5a0f704baa1957a47 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "08b4dd9eba654c9e8faa59ad892a17ab" member_type: VOTER last_known_addr { host: "127.22.50.126" port: 34829 } } peers { permanent_uuid: "5c356dfb31e749c5a0f704baa1957a47" member_type: VOTER last_known_addr { host: "127.22.50.125" port: 39091 } } peers { permanent_uuid: "7d8f4ab9d826436490a94ee04f174e7f" member_type: VOTER last_known_addr { host: "127.22.50.124" port: 43653 } }
20:04:11.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:11.737566 23626 raft_consensus.cc:740] T 00000000000000000000000000000000 P 5c356dfb31e749c5a0f704baa1957a47 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5c356dfb31e749c5a0f704baa1957a47, State: Initialized, Role: FOLLOWER
20:04:11.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:11.738548 23626 consensus_queue.cc:260] T 00000000000000000000000000000000 P 5c356dfb31e749c5a0f704baa1957a47 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "08b4dd9eba654c9e8faa59ad892a17ab" member_type: VOTER last_known_addr { host: "127.22.50.126" port: 34829 } } peers { permanent_uuid: "5c356dfb31e749c5a0f704baa1957a47" member_type: VOTER last_known_addr { host: "127.22.50.125" port: 39091 } } peers { permanent_uuid: "7d8f4ab9d826436490a94ee04f174e7f" member_type: VOTER last_known_addr { host: "127.22.50.124" port: 43653 } }
20:04:11.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:11.741945 23630 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5c356dfb31e749c5a0f704baa1957a47 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "08b4dd9eba654c9e8faa59ad892a17ab" member_type: VOTER last_known_addr { host: "127.22.50.126" port: 34829 } } peers { permanent_uuid: "5c356dfb31e749c5a0f704baa1957a47" member_type: VOTER last_known_addr { host: "127.22.50.125" port: 39091 } } peers { permanent_uuid: "7d8f4ab9d826436490a94ee04f174e7f" member_type: VOTER last_known_addr { host: "127.22.50.124" port: 43653 } } }
20:04:11.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:11.742996 23630 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5c356dfb31e749c5a0f704baa1957a47 [sys.catalog]: This master's current role is: FOLLOWER
20:04:11.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:11.743938 23626 sys_catalog.cc:565] T 00000000000000000000000000000000 P 5c356dfb31e749c5a0f704baa1957a47 [sys.catalog]: configured and running, proceeding with master startup.
20:04:11.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:11.781466 23641 catalog_manager.cc:1269] Loaded cluster ID: 53f04c6e735c41bcaebd2b8fefbc6833
20:04:11.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:11.782094 23641 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 5c356dfb31e749c5a0f704baa1957a47: loading cluster ID for follower catalog manager: success
20:04:11.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:11.794111 23641 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 5c356dfb31e749c5a0f704baa1957a47: acquiring CA information for follower catalog manager: success
20:04:11.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:11.800658 23641 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 5c356dfb31e749c5a0f704baa1957a47: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
20:04:11.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:11.788946 23538 master_runner.cc:186] Error getting master registration for 127.22.50.126:34829: Network error: Client connection negotiation failed: client connection to 127.22.50.126:34829: connect: Connection refused (error 111),
20:04:11.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:11.833263 23538 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:04:11.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:11.841075 23372 heartbeater.cc:344] Connected to a master server at 127.22.50.125:39091
20:04:11.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:11.848209 23591 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "d4d95fecf2ad4ff88f496160b149a076" instance_seqno: 1777838644531830) as {username='kudu', principal='kudu/127.22.50.66@KRBTEST.COM'} at 127.22.50.66:51805; Asking this server to re-register.
20:04:11.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:11.849990 23372 heartbeater.cc:461] Registering TS with master...
20:04:11.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:11.853761 23591 ts_manager.cc:194] Registered new tserver with Master: d4d95fecf2ad4ff88f496160b149a076 (127.22.50.66:32791)
20:04:12.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:04:12.025569 23628 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:04:12.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:04:12.026302 23628 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:04:12.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:04:12.027100 23628 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:04:12.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:04:12.063772 23628 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
20:04:12.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:04:12.064091 23628 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
20:04:12.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:04:12.064592 23628 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:04:12.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:04:12.064947 23628 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
20:04:12.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:04:12.065286 23628 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
20:04:12.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:04:12.065624 23628 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
20:04:12.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:04:12.065876 23628 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:32783/jwks.json
20:04:12.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:12.109967 23628 master_runner.cc:386] Master server non-default flags:
20:04:12.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:04:12.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.50.84:44541
20:04:12.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:04:12.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:04:12.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/master-2/data
20:04:12.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/master-2/wal
20:04:12.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
20:04:12.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:04:12.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.50.126:34829,127.22.50.125:39091,127.22.50.124:43653
20:04:12.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/krb5kdc/kudu.keytab
20:04:12.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.22.50.124
20:04:12.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:04:12.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:04:12.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:04:12.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.50.124:43653
20:04:12.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:04:12.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:04:12.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:04:12.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:32783/jwks.json
20:04:12.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:04:12.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:04:12.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:04:12.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:04:12.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:04:12.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/master-2/data/info.pb
20:04:12.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.50.124
20:04:12.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36701
20:04:12.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:04:12.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:04:12.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:04:12.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:04:12.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:04:12.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/master-2/logs
20:04:12.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:04:12.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:04:12.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:04:12.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:04:12.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:04:12.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0ba7f54e15c9f673bf3e350f30b8d5f1a4037b79
20:04:12.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:04:12.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 03 May 2026 19:43:25 UTC on e7f111948823
20:04:12.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11731
20:04:12.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:04:12.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:12.111729 23628 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:04:12.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:12.112928 23628 file_cache.cc:492] Constructed file cache file cache with capacity 419430
20:04:12.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:04:12.125981 23652 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:12.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:12.201381 23225 heartbeater.cc:344] Connected to a master server at 127.22.50.125:39091
20:04:12.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:12.205528 23591 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "4479bf87dcc14a039b8d139297d6ec9f" instance_seqno: 1777838641747642) as {username='kudu', principal='kudu/127.22.50.65@KRBTEST.COM'} at 127.22.50.65:60625; Asking this server to re-register.
20:04:12.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:12.207333 23225 heartbeater.cc:461] Registering TS with master...
20:04:12.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:12.214151 23591 ts_manager.cc:194] Registered new tserver with Master: 4479bf87dcc14a039b8d139297d6ec9f (127.22.50.65:37573)
20:04:12.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:12.562950 23521 heartbeater.cc:344] Connected to a master server at 127.22.50.125:39091
20:04:12.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:12.563763 23521 heartbeater.cc:499] Master 127.22.50.125:39091 was elected leader, sending a full tablet report...
20:04:12.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:12.569092 23591 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "a84386050e7f4ddf8d0c6b6f64c59135" instance_seqno: 1777838648089108) as {username='kudu', principal='kudu/127.22.50.67@KRBTEST.COM'} at 127.22.50.67:36617; Asking this server to re-register.
20:04:12.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:12.570983 23521 heartbeater.cc:461] Registering TS with master...
20:04:12.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:12.575476 23591 ts_manager.cc:194] Registered new tserver with Master: a84386050e7f4ddf8d0c6b6f64c59135 (127.22.50.67:35917)
20:04:13.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:13.327903 23668 raft_consensus.cc:493] T 00000000000000000000000000000000 P 5c356dfb31e749c5a0f704baa1957a47 [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:04:13.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:13.328682 23668 raft_consensus.cc:515] T 00000000000000000000000000000000 P 5c356dfb31e749c5a0f704baa1957a47 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "08b4dd9eba654c9e8faa59ad892a17ab" member_type: VOTER last_known_addr { host: "127.22.50.126" port: 34829 } } peers { permanent_uuid: "5c356dfb31e749c5a0f704baa1957a47" member_type: VOTER last_known_addr { host: "127.22.50.125" port: 39091 } } peers { permanent_uuid: "7d8f4ab9d826436490a94ee04f174e7f" member_type: VOTER last_known_addr { host: "127.22.50.124" port: 43653 } }
20:04:13.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:04:13.355147 23576 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.22.50.126:34829: connect: Connection refused (error 111)
20:04:13.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:13.363230 23668 leader_election.cc:290] T 00000000000000000000000000000000 P 5c356dfb31e749c5a0f704baa1957a47 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 08b4dd9eba654c9e8faa59ad892a17ab (127.22.50.126:34829), 7d8f4ab9d826436490a94ee04f174e7f (127.22.50.124:43653)
20:04:13.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:04:13.363432 23576 leader_election.cc:336] T 00000000000000000000000000000000 P 5c356dfb31e749c5a0f704baa1957a47 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 08b4dd9eba654c9e8faa59ad892a17ab (127.22.50.126:34829): Network error: Client connection negotiation failed: client connection to 127.22.50.126:34829: connect: Connection refused (error 111)
20:04:13.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:04:13.377250 23577 leader_election.cc:336] T 00000000000000000000000000000000 P 5c356dfb31e749c5a0f704baa1957a47 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 7d8f4ab9d826436490a94ee04f174e7f (127.22.50.124:43653): Network error: Client connection negotiation failed: client connection to 127.22.50.124:43653: connect: Connection refused (error 111)
20:04:13.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:13.378047 23577 leader_election.cc:304] T 00000000000000000000000000000000 P 5c356dfb31e749c5a0f704baa1957a47 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 5c356dfb31e749c5a0f704baa1957a47; no voters: 08b4dd9eba654c9e8faa59ad892a17ab, 7d8f4ab9d826436490a94ee04f174e7f
20:04:13.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:13.380125 23668 raft_consensus.cc:2749] T 00000000000000000000000000000000 P 5c356dfb31e749c5a0f704baa1957a47 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
20:04:14.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:04:13.528648 23651 debug-util.cc:398] Leaking SignalData structure 0x7b0800028d00 after lost signal to thread 23628
20:04:14.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:04:14.058037 23628 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.931s user 0.515s sys 1.138s
20:04:14.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:04:12.127871 23653 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:14.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:04:14.058467 23628 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.932s user 0.515s sys 1.138s
20:04:14.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:04:14.061091 23655 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:14.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:04:14.064455 23654 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1932 milliseconds
20:04:14.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:14.064529 23628 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:04:14.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:14.065902 23628 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:04:14.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:14.068933 23628 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:04:14.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:14.070359 23628 hybrid_clock.cc:648] HybridClock initialized: now 1777838654070142 us; error 231 us; skew 500 ppm
20:04:14.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:14.071362 23628 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:04:14.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 20:04:14 dist-test-slave-b0rl krb5kdc[22741](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777838654, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.50.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:04:14.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:14.078894 23628 init.cc:377] Logged in from keytab as kudu/127.22.50.124@KRBTEST.COM (short username kudu)
20:04:14.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:14.085091 23628 webserver.cc:492] Webserver started at http://127.22.50.124:36701/ using document root <none> and password file <none>
20:04:14.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:14.086126 23628 fs_manager.cc:362] Metadata directory not provided
20:04:14.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:14.086436 23628 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:04:14.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:14.095475 23628 fs_manager.cc:714] Time spent opening directory manager: real 0.006s user 0.005s sys 0.002s
20:04:14.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:14.101325 23681 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:04:14.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:14.102530 23628 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.003s
20:04:14.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:14.102883 23628 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/master-2/data,/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/master-2/wal
20:04:14.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7d8f4ab9d826436490a94ee04f174e7f"
20:04:14.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-03 20:03:54 on dist-test-slave-b0rl"
20:04:14.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:14.104981 23628 fs_report.cc:389] FS layout report
20:04:14.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:04:14.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/master-2/wal
20:04:14.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/master-2/wal
20:04:14.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/master-2/data/data
20:04:14.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:04:14.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:04:14.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:04:14.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:04:14.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:04:14.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:04:14.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:04:14.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:04:14.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:04:14.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:04:14.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:04:14.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:14.162837 23628 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:04:14.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:14.172772 23628 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:04:14.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:14.173290 23628 kserver.cc:163] Server-wide thread pool size limit: 3276
20:04:14.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 20:04:14 dist-test-slave-b0rl krb5kdc[22741](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777838654, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.50.66@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:04:14.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:14.229079 23254 init.cc:295] Successfully reacquired a new kerberos TGT
20:04:14.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:14.264062 23628 rpc_server.cc:307] RPC server started. Bound to: 127.22.50.124:43653
20:04:14.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:14.264099 23733 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.22.50.124:43653 every 8 connection(s)
20:04:14.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:14.268129 23628 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/master-2/data/info.pb
20:04:14.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:14.270665 22729 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskqmogBp/build/tsan/bin/kudu as pid 23628
20:04:14.272 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:04:14.272 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.22.50.126:34829
20:04:14.273 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
20:04:14.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:14.275537 22729 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskqmogBp/build/tsan/bin/kudu
20:04:14.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskqmogBp/build/tsan/bin/kudu
20:04:14.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/master-0/wal
20:04:14.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/master-0/data
20:04:14.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:04:14.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:04:14.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:04:14.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:04:14.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:04:14.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:04:14.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/master-0/logs
20:04:14.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/master-0/data/info.pb
20:04:14.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:04:14.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:04:14.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:04:14.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:04:14.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:04:14.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:04:14.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:04:14.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:04:14.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:04:14.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:04:14.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:04:14.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:04:14.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.50.126:34829
20:04:14.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.50.126
20:04:14.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=40523
20:04:14.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/krb5kdc/kudu.keytab
20:04:14.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.22.50.126
20:04:14.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:04:14.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:04:14.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:04:14.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
20:04:14.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.50.84:44541
20:04:14.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:04:14.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:04:14.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:04:14.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:04:14.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.50.126:34829,127.22.50.125:39091,127.22.50.124:43653
20:04:14.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:04:14.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:32783/jwks.json
20:04:14.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
20:04:14.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:04:14.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:14.284518 23734 sys_catalog.cc:263] Verifying existing consensus state
20:04:14.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:14.294238 23734 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 7d8f4ab9d826436490a94ee04f174e7f: Bootstrap starting.
20:04:14.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:14.349399 23734 log.cc:826] T 00000000000000000000000000000000 P 7d8f4ab9d826436490a94ee04f174e7f: Log is configured to *not* fsync() on all Append() calls
20:04:14.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:14.367151 23734 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 7d8f4ab9d826436490a94ee04f174e7f: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
20:04:14.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:14.368192 23734 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 7d8f4ab9d826436490a94ee04f174e7f: Bootstrap complete.
20:04:14.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:14.394124 23734 raft_consensus.cc:359] T 00000000000000000000000000000000 P 7d8f4ab9d826436490a94ee04f174e7f [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "08b4dd9eba654c9e8faa59ad892a17ab" member_type: VOTER last_known_addr { host: "127.22.50.126" port: 34829 } } peers { permanent_uuid: "5c356dfb31e749c5a0f704baa1957a47" member_type: VOTER last_known_addr { host: "127.22.50.125" port: 39091 } } peers { permanent_uuid: "7d8f4ab9d826436490a94ee04f174e7f" member_type: VOTER last_known_addr { host: "127.22.50.124" port: 43653 } }
20:04:14.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:14.397588 23734 raft_consensus.cc:740] T 00000000000000000000000000000000 P 7d8f4ab9d826436490a94ee04f174e7f [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7d8f4ab9d826436490a94ee04f174e7f, State: Initialized, Role: FOLLOWER
20:04:14.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:14.398629 23734 consensus_queue.cc:260] T 00000000000000000000000000000000 P 7d8f4ab9d826436490a94ee04f174e7f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "08b4dd9eba654c9e8faa59ad892a17ab" member_type: VOTER last_known_addr { host: "127.22.50.126" port: 34829 } } peers { permanent_uuid: "5c356dfb31e749c5a0f704baa1957a47" member_type: VOTER last_known_addr { host: "127.22.50.125" port: 39091 } } peers { permanent_uuid: "7d8f4ab9d826436490a94ee04f174e7f" member_type: VOTER last_known_addr { host: "127.22.50.124" port: 43653 } }
20:04:14.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:14.415824 23738 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7d8f4ab9d826436490a94ee04f174e7f [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "08b4dd9eba654c9e8faa59ad892a17ab" member_type: VOTER last_known_addr { host: "127.22.50.126" port: 34829 } } peers { permanent_uuid: "5c356dfb31e749c5a0f704baa1957a47" member_type: VOTER last_known_addr { host: "127.22.50.125" port: 39091 } } peers { permanent_uuid: "7d8f4ab9d826436490a94ee04f174e7f" member_type: VOTER last_known_addr { host: "127.22.50.124" port: 43653 } } }
20:04:14.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:14.425611 23738 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7d8f4ab9d826436490a94ee04f174e7f [sys.catalog]: This master's current role is: FOLLOWER
20:04:14.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:14.425915 23734 sys_catalog.cc:565] T 00000000000000000000000000000000 P 7d8f4ab9d826436490a94ee04f174e7f [sys.catalog]: configured and running, proceeding with master startup.
20:04:14.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:04:14.463620 23622 debug-util.cc:398] Leaking SignalData structure 0x7b08000711c0 after lost signal to thread 23541
20:04:14.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:04:14.464280 23622 debug-util.cc:398] Leaking SignalData structure 0x7b0800071ee0 after lost signal to thread 23625
20:04:14.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:14.614964 23520 heartbeater.cc:344] Connected to a master server at 127.22.50.124:43653
20:04:14.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:14.622249 23699 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "a84386050e7f4ddf8d0c6b6f64c59135" instance_seqno: 1777838648089108) as {username='kudu', principal='kudu/127.22.50.67@KRBTEST.COM'} at 127.22.50.67:47677; Asking this server to re-register.
20:04:14.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:14.624557 23520 heartbeater.cc:461] Registering TS with master...
20:04:14.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:14.629484 23698 ts_manager.cc:194] Registered new tserver with Master: a84386050e7f4ddf8d0c6b6f64c59135 (127.22.50.67:35917)
20:04:14.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:14.665776 23752 catalog_manager.cc:1269] Loaded cluster ID: 53f04c6e735c41bcaebd2b8fefbc6833
20:04:14.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:14.669776 23752 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 7d8f4ab9d826436490a94ee04f174e7f: loading cluster ID for follower catalog manager: success
20:04:14.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:14.677920 23752 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 7d8f4ab9d826436490a94ee04f174e7f: acquiring CA information for follower catalog manager: success
20:04:14.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:14.632083 23628 master_runner.cc:186] Error getting master registration for 127.22.50.126:34829: Network error: Client connection negotiation failed: client connection to 127.22.50.126:34829: connect: Connection refused (error 111),
20:04:14.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:14.703238 23628 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:04:14.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:14.707876 23752 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 7d8f4ab9d826436490a94ee04f174e7f: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
20:04:14.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:04:14.853807 23736 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:04:14.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:04:14.854619 23736 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:04:14.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:04:14.855687 23736 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:04:14.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:14.890738 23371 heartbeater.cc:344] Connected to a master server at 127.22.50.124:43653
20:04:14.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:14.894058 23699 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "d4d95fecf2ad4ff88f496160b149a076" instance_seqno: 1777838644531830) as {username='kudu', principal='kudu/127.22.50.66@KRBTEST.COM'} at 127.22.50.66:39941; Asking this server to re-register.
20:04:14.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:14.898015 23371 heartbeater.cc:461] Registering TS with master...
20:04:14.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:14.904114 23699 ts_manager.cc:194] Registered new tserver with Master: d4d95fecf2ad4ff88f496160b149a076 (127.22.50.66:32791)
20:04:14.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:14.912678 23761 raft_consensus.cc:493] T 00000000000000000000000000000000 P 5c356dfb31e749c5a0f704baa1957a47 [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:04:14.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:14.913594 23761 raft_consensus.cc:515] T 00000000000000000000000000000000 P 5c356dfb31e749c5a0f704baa1957a47 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "08b4dd9eba654c9e8faa59ad892a17ab" member_type: VOTER last_known_addr { host: "127.22.50.126" port: 34829 } } peers { permanent_uuid: "5c356dfb31e749c5a0f704baa1957a47" member_type: VOTER last_known_addr { host: "127.22.50.125" port: 39091 } } peers { permanent_uuid: "7d8f4ab9d826436490a94ee04f174e7f" member_type: VOTER last_known_addr { host: "127.22.50.124" port: 43653 } }
20:04:14.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:14.928046 23761 leader_election.cc:290] T 00000000000000000000000000000000 P 5c356dfb31e749c5a0f704baa1957a47 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 08b4dd9eba654c9e8faa59ad892a17ab (127.22.50.126:34829), 7d8f4ab9d826436490a94ee04f174e7f (127.22.50.124:43653)
20:04:14.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:04:14.944387 23736 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
20:04:14.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:04:14.944730 23736 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
20:04:14.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:04:14.945358 23736 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:04:14.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:04:14.945727 23736 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
20:04:14.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:04:14.946055 23736 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
20:04:14.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:04:14.946388 23736 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
20:04:14.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:04:14.945890 23576 leader_election.cc:336] T 00000000000000000000000000000000 P 5c356dfb31e749c5a0f704baa1957a47 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 08b4dd9eba654c9e8faa59ad892a17ab (127.22.50.126:34829): Network error: Client connection negotiation failed: client connection to 127.22.50.126:34829: connect: Connection refused (error 111)
20:04:14.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:04:14.946671 23736 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:32783/jwks.json
20:04:14.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:14.976807 23709 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "5c356dfb31e749c5a0f704baa1957a47" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "7d8f4ab9d826436490a94ee04f174e7f" is_pre_election: true
20:04:14.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:14.977803 23709 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 7d8f4ab9d826436490a94ee04f174e7f [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 5c356dfb31e749c5a0f704baa1957a47 in term 1.
20:04:14.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:14.979818 23577 leader_election.cc:304] T 00000000000000000000000000000000 P 5c356dfb31e749c5a0f704baa1957a47 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 5c356dfb31e749c5a0f704baa1957a47, 7d8f4ab9d826436490a94ee04f174e7f; no voters: 08b4dd9eba654c9e8faa59ad892a17ab
20:04:14.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:14.980621 23761 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 5c356dfb31e749c5a0f704baa1957a47 [term 1 FOLLOWER]: Leader pre-election won for term 2
20:04:14.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:14.980937 23761 raft_consensus.cc:493] T 00000000000000000000000000000000 P 5c356dfb31e749c5a0f704baa1957a47 [term 1 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
20:04:14.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:14.981246 23761 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 5c356dfb31e749c5a0f704baa1957a47 [term 1 FOLLOWER]: Advancing to term 2
20:04:14.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:14.997542 23761 raft_consensus.cc:515] T 00000000000000000000000000000000 P 5c356dfb31e749c5a0f704baa1957a47 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "08b4dd9eba654c9e8faa59ad892a17ab" member_type: VOTER last_known_addr { host: "127.22.50.126" port: 34829 } } peers { permanent_uuid: "5c356dfb31e749c5a0f704baa1957a47" member_type: VOTER last_known_addr { host: "127.22.50.125" port: 39091 } } peers { permanent_uuid: "7d8f4ab9d826436490a94ee04f174e7f" member_type: VOTER last_known_addr { host: "127.22.50.124" port: 43653 } }
20:04:15.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:04:15.018265 23576 leader_election.cc:336] T 00000000000000000000000000000000 P 5c356dfb31e749c5a0f704baa1957a47 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 08b4dd9eba654c9e8faa59ad892a17ab (127.22.50.126:34829): Network error: Client connection negotiation failed: client connection to 127.22.50.126:34829: connect: Connection refused (error 111)
20:04:15.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:15.020787 23761 leader_election.cc:290] T 00000000000000000000000000000000 P 5c356dfb31e749c5a0f704baa1957a47 [CANDIDATE]: Term 2 election: Requested vote from peers 08b4dd9eba654c9e8faa59ad892a17ab (127.22.50.126:34829), 7d8f4ab9d826436490a94ee04f174e7f (127.22.50.124:43653)
20:04:15.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:15.022076 23709 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "5c356dfb31e749c5a0f704baa1957a47" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "7d8f4ab9d826436490a94ee04f174e7f"
20:04:15.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:15.022225 23736 master_runner.cc:386] Master server non-default flags:
20:04:15.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:04:15.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.50.84:44541
20:04:15.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:04:15.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:04:15.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/master-0/data
20:04:15.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/master-0/wal
20:04:15.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
20:04:15.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:04:15.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.50.126:34829,127.22.50.125:39091,127.22.50.124:43653
20:04:15.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/krb5kdc/kudu.keytab
20:04:15.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.22.50.126
20:04:15.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:04:15.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:04:15.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:04:15.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.50.126:34829
20:04:15.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:04:15.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:04:15.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:04:15.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:32783/jwks.json
20:04:15.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:04:15.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:04:15.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:04:15.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:04:15.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:04:15.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/master-0/data/info.pb
20:04:15.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.50.126
20:04:15.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=40523
20:04:15.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:04:15.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:04:15.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:04:15.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:04:15.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:04:15.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/master-0/logs
20:04:15.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:04:15.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:04:15.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:04:15.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:04:15.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:04:15.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0ba7f54e15c9f673bf3e350f30b8d5f1a4037b79
20:04:15.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:04:15.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 03 May 2026 19:43:25 UTC on e7f111948823
20:04:15.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11731
20:04:15.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:04:15.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:15.022682 23709 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 7d8f4ab9d826436490a94ee04f174e7f [term 1 FOLLOWER]: Advancing to term 2
20:04:15.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:15.024024 23736 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:04:15.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:15.025527 23736 file_cache.cc:492] Constructed file cache file cache with capacity 419430
20:04:15.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:15.037384 23709 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 7d8f4ab9d826436490a94ee04f174e7f [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 5c356dfb31e749c5a0f704baa1957a47 in term 2.
20:04:15.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:15.042096 23577 leader_election.cc:304] T 00000000000000000000000000000000 P 5c356dfb31e749c5a0f704baa1957a47 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 5c356dfb31e749c5a0f704baa1957a47, 7d8f4ab9d826436490a94ee04f174e7f; no voters: 08b4dd9eba654c9e8faa59ad892a17ab
20:04:15.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:15.043426 23761 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 5c356dfb31e749c5a0f704baa1957a47 [term 2 FOLLOWER]: Leader election won for term 2
20:04:15.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:04:15.056588 23766 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:15.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:15.070030 23761 raft_consensus.cc:697] T 00000000000000000000000000000000 P 5c356dfb31e749c5a0f704baa1957a47 [term 2 LEADER]: Becoming Leader. State: Replica: 5c356dfb31e749c5a0f704baa1957a47, State: Running, Role: LEADER
20:04:15.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:15.071328 23761 consensus_queue.cc:237] T 00000000000000000000000000000000 P 5c356dfb31e749c5a0f704baa1957a47 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "08b4dd9eba654c9e8faa59ad892a17ab" member_type: VOTER last_known_addr { host: "127.22.50.126" port: 34829 } } peers { permanent_uuid: "5c356dfb31e749c5a0f704baa1957a47" member_type: VOTER last_known_addr { host: "127.22.50.125" port: 39091 } } peers { permanent_uuid: "7d8f4ab9d826436490a94ee04f174e7f" member_type: VOTER last_known_addr { host: "127.22.50.124" port: 43653 } }
20:04:15.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:15.091825 23767 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5c356dfb31e749c5a0f704baa1957a47 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 5c356dfb31e749c5a0f704baa1957a47. Latest consensus state: current_term: 2 leader_uuid: "5c356dfb31e749c5a0f704baa1957a47" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "08b4dd9eba654c9e8faa59ad892a17ab" member_type: VOTER last_known_addr { host: "127.22.50.126" port: 34829 } } peers { permanent_uuid: "5c356dfb31e749c5a0f704baa1957a47" member_type: VOTER last_known_addr { host: "127.22.50.125" port: 39091 } } peers { permanent_uuid: "7d8f4ab9d826436490a94ee04f174e7f" member_type: VOTER last_known_addr { host: "127.22.50.124" port: 43653 } } }
20:04:15.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:15.093214 23767 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5c356dfb31e749c5a0f704baa1957a47 [sys.catalog]: This master's current role is: LEADER
20:04:15.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:15.105614 23773 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
20:04:15.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:15.124882 23773 catalog_manager.cc:1494] Initializing Kudu cluster ID...
20:04:15.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:15.130419 23773 catalog_manager.cc:1269] Loaded cluster ID: 53f04c6e735c41bcaebd2b8fefbc6833
20:04:15.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:15.135128 23773 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
20:04:15.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:15.145301 23773 catalog_manager.cc:1514] Loading token signing keys...
20:04:15.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:15.150111 23773 catalog_manager.cc:6055] T 00000000000000000000000000000000 P 5c356dfb31e749c5a0f704baa1957a47: Loaded TSK: 0
20:04:15.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:15.156507 23773 catalog_manager.cc:1524] Initializing in-progress tserver states...
20:04:15.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:15.246567 23224 heartbeater.cc:344] Connected to a master server at 127.22.50.124:43653
20:04:15.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:15.250030 23699 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "4479bf87dcc14a039b8d139297d6ec9f" instance_seqno: 1777838641747642) as {username='kudu', principal='kudu/127.22.50.65@KRBTEST.COM'} at 127.22.50.65:59627; Asking this server to re-register.
20:04:15.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:15.255573 23224 heartbeater.cc:461] Registering TS with master...
20:04:15.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:15.259303 23699 ts_manager.cc:194] Registered new tserver with Master: 4479bf87dcc14a039b8d139297d6ec9f (127.22.50.65:37573)
20:04:15.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:15.497641 23709 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 7d8f4ab9d826436490a94ee04f174e7f [term 2 FOLLOWER]: Refusing update from remote peer 5c356dfb31e749c5a0f704baa1957a47: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 2 index: 5. (index mismatch)
20:04:15.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:15.499763 23767 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 5c356dfb31e749c5a0f704baa1957a47 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7d8f4ab9d826436490a94ee04f174e7f" member_type: VOTER last_known_addr { host: "127.22.50.124" port: 43653 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
20:04:15.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:15.530474 23779 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7d8f4ab9d826436490a94ee04f174e7f [sys.catalog]: SysCatalogTable state changed. Reason: New leader 5c356dfb31e749c5a0f704baa1957a47. Latest consensus state: current_term: 2 leader_uuid: "5c356dfb31e749c5a0f704baa1957a47" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "08b4dd9eba654c9e8faa59ad892a17ab" member_type: VOTER last_known_addr { host: "127.22.50.126" port: 34829 } } peers { permanent_uuid: "5c356dfb31e749c5a0f704baa1957a47" member_type: VOTER last_known_addr { host: "127.22.50.125" port: 39091 } } peers { permanent_uuid: "7d8f4ab9d826436490a94ee04f174e7f" member_type: VOTER last_known_addr { host: "127.22.50.124" port: 43653 } } }
20:04:15.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:15.531783 23779 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7d8f4ab9d826436490a94ee04f174e7f [sys.catalog]: This master's current role is: FOLLOWER
20:04:15.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:15.562182 23761 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5c356dfb31e749c5a0f704baa1957a47 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "5c356dfb31e749c5a0f704baa1957a47" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "08b4dd9eba654c9e8faa59ad892a17ab" member_type: VOTER last_known_addr { host: "127.22.50.126" port: 34829 } } peers { permanent_uuid: "5c356dfb31e749c5a0f704baa1957a47" member_type: VOTER last_known_addr { host: "127.22.50.125" port: 39091 } } peers { permanent_uuid: "7d8f4ab9d826436490a94ee04f174e7f" member_type: VOTER last_known_addr { host: "127.22.50.124" port: 43653 } } }
20:04:15.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:15.563037 23761 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5c356dfb31e749c5a0f704baa1957a47 [sys.catalog]: This master's current role is: LEADER
20:04:15.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:15.566562 23779 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7d8f4ab9d826436490a94ee04f174e7f [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "5c356dfb31e749c5a0f704baa1957a47" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "08b4dd9eba654c9e8faa59ad892a17ab" member_type: VOTER last_known_addr { host: "127.22.50.126" port: 34829 } } peers { permanent_uuid: "5c356dfb31e749c5a0f704baa1957a47" member_type: VOTER last_known_addr { host: "127.22.50.125" port: 39091 } } peers { permanent_uuid: "7d8f4ab9d826436490a94ee04f174e7f" member_type: VOTER last_known_addr { host: "127.22.50.124" port: 43653 } } }
20:04:15.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:15.567476 23779 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7d8f4ab9d826436490a94ee04f174e7f [sys.catalog]: This master's current role is: FOLLOWER
20:04:15.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:04:15.597776 23576 consensus_peers.cc:597] T 00000000000000000000000000000000 P 5c356dfb31e749c5a0f704baa1957a47 -> Peer 08b4dd9eba654c9e8faa59ad892a17ab (127.22.50.126:34829): Couldn't send request to peer 08b4dd9eba654c9e8faa59ad892a17ab. Status: Network error: Client connection negotiation failed: client connection to 127.22.50.126:34829: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
20:04:16.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:16.246166 23225 heartbeater.cc:499] Master 127.22.50.125:39091 was elected leader, sending a full tablet report...
20:04:16.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:16.596750 23521 heartbeater.cc:499] Master 127.22.50.125:39091 was elected leader, sending a full tablet report...
20:04:16.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:04:15.067488 23768 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:16.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:15.076896 23736 server_base.cc:1061] running on GCE node
20:04:16.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:04:15.074999 23771 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:04:16.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:16.804169 23736 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:04:16.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:16.811805 23736 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:04:16.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:16.820016 23736 hybrid_clock.cc:648] HybridClock initialized: now 1777838656819912 us; error 114 us; skew 500 ppm
20:04:16.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:16.821419 23736 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:04:16.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 20:04:16 dist-test-slave-b0rl krb5kdc[22741](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777838656, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.50.126@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:04:16.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:16.832370 23736 init.cc:377] Logged in from keytab as kudu/127.22.50.126@KRBTEST.COM (short username kudu)
20:04:16.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:16.842427 23736 webserver.cc:492] Webserver started at http://127.22.50.126:40523/ using document root <none> and password file <none>
20:04:16.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:16.843680 23736 fs_manager.cc:362] Metadata directory not provided
20:04:16.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:16.844031 23736 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:04:16.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:16.848275 23538 master_runner.cc:186] Error getting master registration for 127.22.50.126:34829: Network error: Client connection negotiation failed: client connection to 127.22.50.126:34829: connect: Connection refused (error 111),
20:04:16.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:16.848718 23538 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:04:16.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:16.855044 23736 fs_manager.cc:714] Time spent opening directory manager: real 0.007s user 0.002s sys 0.005s
20:04:16.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:16.863848 23795 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:04:16.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:16.865379 23736 fs_manager.cc:730] Time spent opening block manager: real 0.008s user 0.003s sys 0.006s
20:04:16.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:16.865897 23736 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/master-0/data,/tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/master-0/wal
20:04:16.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "08b4dd9eba654c9e8faa59ad892a17ab"
20:04:16.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-03 20:03:49 on dist-test-slave-b0rl"
20:04:16.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:16.868541 23736 fs_report.cc:389] FS layout report
20:04:16.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:04:16.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/master-0/wal
20:04:16.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/master-0/wal
20:04:16.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/master-0/data/data
20:04:16.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:04:16.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:04:16.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:04:16.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:04:16.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:04:16.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:04:16.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:04:16.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:04:16.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:04:16.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:04:16.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:04:16.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:16.875005 23372 heartbeater.cc:499] Master 127.22.50.125:39091 was elected leader, sending a full tablet report...
20:04:16.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:16.952510 23736 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:04:16.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:16.960501 23736 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:04:16.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:16.961123 23736 kserver.cc:163] Server-wide thread pool size limit: 3276
20:04:17.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:17.088114 23736 rpc_server.cc:307] RPC server started. Bound to: 127.22.50.126:34829
20:04:17.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:17.088258 23847 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.22.50.126:34829 every 8 connection(s)
20:04:17.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:17.092132 23736 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskqmogBp/test-tmp/mini-kudu-cluster7394091750350669132/master-0/data/info.pb
20:04:17.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:17.098219 22729 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskqmogBp/build/tsan/bin/kudu as pid 23736
20:04:17.099 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:04:17.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:17.111984 23848 sys_catalog.cc:263] Verifying existing consensus state
20:04:17.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:17.120675 23848 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 08b4dd9eba654c9e8faa59ad892a17ab: Bootstrap starting.
20:04:17.177 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.22.50.125:39091(127.22.50.125:39091)] connecting to peer
20:04:17.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:17.205758 23848 log.cc:826] T 00000000000000000000000000000000 P 08b4dd9eba654c9e8faa59ad892a17ab: Log is configured to *not* fsync() on all Append() calls
20:04:17.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:17.235407 23848 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 08b4dd9eba654c9e8faa59ad892a17ab: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
20:04:17.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:17.237012 23848 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 08b4dd9eba654c9e8faa59ad892a17ab: Bootstrap complete.
20:04:17.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:17.317483 23848 raft_consensus.cc:359] T 00000000000000000000000000000000 P 08b4dd9eba654c9e8faa59ad892a17ab [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "08b4dd9eba654c9e8faa59ad892a17ab" member_type: VOTER last_known_addr { host: "127.22.50.126" port: 34829 } } peers { permanent_uuid: "5c356dfb31e749c5a0f704baa1957a47" member_type: VOTER last_known_addr { host: "127.22.50.125" port: 39091 } } peers { permanent_uuid: "7d8f4ab9d826436490a94ee04f174e7f" member_type: VOTER last_known_addr { host: "127.22.50.124" port: 43653 } }
20:04:17.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:17.320926 23848 raft_consensus.cc:740] T 00000000000000000000000000000000 P 08b4dd9eba654c9e8faa59ad892a17ab [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 08b4dd9eba654c9e8faa59ad892a17ab, State: Initialized, Role: FOLLOWER
20:04:17.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:17.328003 23848 consensus_queue.cc:260] T 00000000000000000000000000000000 P 08b4dd9eba654c9e8faa59ad892a17ab [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "08b4dd9eba654c9e8faa59ad892a17ab" member_type: VOTER last_known_addr { host: "127.22.50.126" port: 34829 } } peers { permanent_uuid: "5c356dfb31e749c5a0f704baa1957a47" member_type: VOTER last_known_addr { host: "127.22.50.125" port: 39091 } } peers { permanent_uuid: "7d8f4ab9d826436490a94ee04f174e7f" member_type: VOTER last_known_addr { host: "127.22.50.124" port: 43653 } }
20:04:17.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 20:04:17 dist-test-slave-b0rl krb5kdc[22741](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777838651, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.50.65@KRBTEST.COM for kudu/127.22.50.126@KRBTEST.COM
20:04:17.336 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.22.50.124:43653(127.22.50.124:43653)] connecting to peer
20:04:17.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:17.339828 23855 sys_catalog.cc:455] T 00000000000000000000000000000000 P 08b4dd9eba654c9e8faa59ad892a17ab [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "08b4dd9eba654c9e8faa59ad892a17ab" member_type: VOTER last_known_addr { host: "127.22.50.126" port: 34829 } } peers { permanent_uuid: "5c356dfb31e749c5a0f704baa1957a47" member_type: VOTER last_known_addr { host: "127.22.50.125" port: 39091 } } peers { permanent_uuid: "7d8f4ab9d826436490a94ee04f174e7f" member_type: VOTER last_known_addr { host: "127.22.50.124" port: 43653 } } }
20:04:17.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:17.340770 23855 sys_catalog.cc:458] T 00000000000000000000000000000000 P 08b4dd9eba654c9e8faa59ad892a17ab [sys.catalog]: This master's current role is: FOLLOWER
20:04:17.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:17.343894 23848 sys_catalog.cc:565] T 00000000000000000000000000000000 P 08b4dd9eba654c9e8faa59ad892a17ab [sys.catalog]: configured and running, proceeding with master startup.
20:04:17.351 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.22.50.126:34829(127.22.50.126:34829)] connecting to peer
20:04:17.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:17.383292 23229 heartbeater.cc:344] Connected to a master server at 127.22.50.126:34829
20:04:17.385 [DEBUG - kudu-nio-2] (Connection.java:757) [peer master-127.22.50.126:34829(127.22.50.126:34829)] Successfully connected to peer
20:04:17.387 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.22.50.125:39091(127.22.50.125:39091)] Successfully connected to peer
20:04:17.390 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.22.50.124:43653(127.22.50.124:43653)] Successfully connected to peer
20:04:17.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 20:04:17 dist-test-slave-b0rl krb5kdc[22741](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777838651, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.50.125@KRBTEST.COM for kudu/127.22.50.126@KRBTEST.COM
20:04:17.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:17.468078 23823 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 08b4dd9eba654c9e8faa59ad892a17ab [term 1 FOLLOWER]: Advancing to term 2
20:04:17.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:17.568145 23855 sys_catalog.cc:455] T 00000000000000000000000000000000 P 08b4dd9eba654c9e8faa59ad892a17ab [sys.catalog]: SysCatalogTable state changed. Reason: New leader 5c356dfb31e749c5a0f704baa1957a47. Latest consensus state: current_term: 2 leader_uuid: "5c356dfb31e749c5a0f704baa1957a47" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "08b4dd9eba654c9e8faa59ad892a17ab" member_type: VOTER last_known_addr { host: "127.22.50.126" port: 34829 } } peers { permanent_uuid: "5c356dfb31e749c5a0f704baa1957a47" member_type: VOTER last_known_addr { host: "127.22.50.125" port: 39091 } } peers { permanent_uuid: "7d8f4ab9d826436490a94ee04f174e7f" member_type: VOTER last_known_addr { host: "127.22.50.124" port: 43653 } } }
20:04:17.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:17.568907 23855 sys_catalog.cc:458] T 00000000000000000000000000000000 P 08b4dd9eba654c9e8faa59ad892a17ab [sys.catalog]: This master's current role is: FOLLOWER
20:04:17.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:17.571739 23870 sys_catalog.cc:455] T 00000000000000000000000000000000 P 08b4dd9eba654c9e8faa59ad892a17ab [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "5c356dfb31e749c5a0f704baa1957a47" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "08b4dd9eba654c9e8faa59ad892a17ab" member_type: VOTER last_known_addr { host: "127.22.50.126" port: 34829 } } peers { permanent_uuid: "5c356dfb31e749c5a0f704baa1957a47" member_type: VOTER last_known_addr { host: "127.22.50.125" port: 39091 } } peers { permanent_uuid: "7d8f4ab9d826436490a94ee04f174e7f" member_type: VOTER last_known_addr { host: "127.22.50.124" port: 43653 } } }
20:04:17.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:17.572610 23870 sys_catalog.cc:458] T 00000000000000000000000000000000 P 08b4dd9eba654c9e8faa59ad892a17ab [sys.catalog]: This master's current role is: FOLLOWER
20:04:17.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 20:04:17 dist-test-slave-b0rl krb5kdc[22741](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777838657, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.50.67@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:04:17.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:17.588236 23402 init.cc:295] Successfully reacquired a new kerberos TGT
20:04:17.644 [WARN - kudu-nio-1] (SecurityContext.java:206) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
20:04:17.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:17.644660 23875 catalog_manager.cc:805] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
20:04:17.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 20:04:17 dist-test-slave-b0rl krb5kdc[22741](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777838657, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.50.67@KRBTEST.COM for kudu/127.22.50.126@KRBTEST.COM
20:04:17.682 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.22.50.125
20:04:17.684 [DEBUG - kudu-nio-2] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.22.50.126
20:04:17.685 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.22.50.124
20:04:17.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:17.694396 23525 heartbeater.cc:344] Connected to a master server at 127.22.50.126:34829
20:04:17.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:17.703830 23813 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "a84386050e7f4ddf8d0c6b6f64c59135" instance_seqno: 1777838648089108) as {username='kudu', principal='kudu/127.22.50.67@KRBTEST.COM'} at 127.22.50.67:54019; Asking this server to re-register.
20:04:17.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:17.706854 23525 heartbeater.cc:461] Registering TS with master...
20:04:17.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:17.711033 23813 ts_manager.cc:194] Registered new tserver with Master: a84386050e7f4ddf8d0c6b6f64c59135 (127.22.50.67:35917)
20:04:17.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 20:04:17 dist-test-slave-b0rl krb5kdc[22741](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777838656, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.50.126@KRBTEST.COM for kudu/127.22.50.126@KRBTEST.COM
20:04:17.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:17.769479 23767 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5c356dfb31e749c5a0f704baa1957a47 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "5c356dfb31e749c5a0f704baa1957a47" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "08b4dd9eba654c9e8faa59ad892a17ab" member_type: VOTER last_known_addr { host: "127.22.50.126" port: 34829 } } peers { permanent_uuid: "5c356dfb31e749c5a0f704baa1957a47" member_type: VOTER last_known_addr { host: "127.22.50.125" port: 39091 } } peers { permanent_uuid: "7d8f4ab9d826436490a94ee04f174e7f" member_type: VOTER last_known_addr { host: "127.22.50.124" port: 43653 } } }
20:04:17.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:17.771307 23767 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5c356dfb31e749c5a0f704baa1957a47 [sys.catalog]: This master's current role is: LEADER
20:04:17.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 20:04:17 dist-test-slave-b0rl krb5kdc[22741](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777838656, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.50.126@KRBTEST.COM for kudu/127.22.50.125@KRBTEST.COM
20:04:17.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 20:04:17 dist-test-slave-b0rl krb5kdc[22741](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777838656, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.50.126@KRBTEST.COM for kudu/127.22.50.124@KRBTEST.COM
20:04:17.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 20:04:17 dist-test-slave-b0rl krb5kdc[22741](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777838654, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.50.66@KRBTEST.COM for kudu/127.22.50.126@KRBTEST.COM
20:04:18.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:18.009954 23373 heartbeater.cc:344] Connected to a master server at 127.22.50.126:34829
20:04:18.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:18.013154 23813 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "d4d95fecf2ad4ff88f496160b149a076" instance_seqno: 1777838644531830) as {username='kudu', principal='kudu/127.22.50.66@KRBTEST.COM'} at 127.22.50.66:49075; Asking this server to re-register.
20:04:18.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:18.017205 23373 heartbeater.cc:461] Registering TS with master...
20:04:18.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:18.020676 23813 ts_manager.cc:194] Registered new tserver with Master: d4d95fecf2ad4ff88f496160b149a076 (127.22.50.66:32791)
20:04:18.130 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.50.124:43653(127.22.50.124:43653)] cleaning up while in state NEGOTIATING due to: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
20:04:18.130 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.22.50.126:34829(127.22.50.126:34829)] cleaning up while in state NEGOTIATING due to: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
20:04:18.131 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.22.50.126:34829: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
20:04:18.131 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.50.124:43653: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
20:04:18.135 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.50.125:39091(127.22.50.125:39091)] cleaning up while in state NEGOTIATING due to: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
20:04:18.135 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.50.125:39091: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
20:04:18.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:04:18.138401 23860 negotiation.cc:336] Failed RPC negotiation. Trace:
20:04:18.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 20:04:17.384804 (+ 0us) reactor.cc:678] Submitting negotiation task for server connection from 127.0.0.1:60186 (local address 127.22.50.126:34829)
20:04:18.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 20:04:17.408122 (+ 23318us) server_negotiation.cc:207] Beginning negotiation
20:04:18.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 20:04:17.408152 (+ 30us) server_negotiation.cc:400] Waiting for connection header
20:04:18.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 20:04:17.507273 (+ 99121us) server_negotiation.cc:408] Connection header received
20:04:18.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 20:04:17.602200 (+ 94927us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
20:04:18.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 20:04:17.602238 (+ 38us) server_negotiation.cc:462] Received NEGOTIATE request from client
20:04:18.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 20:04:17.602467 (+ 229us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
20:04:18.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 20:04:17.602662 (+ 195us) server_negotiation.cc:227] Negotiated authn=SASL
20:04:18.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 20:04:18.000149 (+397487us) server_negotiation.cc:366] Received TLS_HANDSHAKE NegotiatePB request
20:04:18.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 20:04:18.002131 (+ 1982us) server_negotiation.cc:378] Sending TLS_HANDSHAKE NegotiatePB response
20:04:18.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 20:04:18.118896 (+116765us) server_negotiation.cc:366] Received TLS_HANDSHAKE NegotiatePB request
20:04:18.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 20:04:18.119468 (+ 572us) server_negotiation.cc:663] Negotiated TLSv1.3 with cipher TLS_AES_128_GCM_SHA256 TLSv1.3 Kx=any Au=any Enc=AESGCM(128) Mac=AEAD
20:04:18.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 20:04:18.136816 (+ 17348us) negotiation.cc:326] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:60186: BlockingRecv error: failed to read from TLS socket (remote: 127.0.0.1:60186): Cannot send after transport endpoint shutdown (error 108)
20:04:18.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":21330}
20:04:18.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:04:18.139292 23862 negotiation.cc:336] Failed RPC negotiation. Trace:
20:04:18.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 20:04:17.389886 (+ 0us) reactor.cc:678] Submitting negotiation task for server connection from 127.0.0.1:45618 (local address 127.22.50.124:43653)
20:04:18.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 20:04:17.405165 (+ 15279us) server_negotiation.cc:207] Beginning negotiation
20:04:18.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 20:04:17.405197 (+ 32us) server_negotiation.cc:400] Waiting for connection header
20:04:18.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 20:04:17.506679 (+101482us) server_negotiation.cc:408] Connection header received
20:04:18.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 20:04:17.600397 (+ 93718us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
20:04:18.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 20:04:17.600448 (+ 51us) server_negotiation.cc:462] Received NEGOTIATE request from client
20:04:18.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 20:04:17.600712 (+ 264us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
20:04:18.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 20:04:17.600922 (+ 210us) server_negotiation.cc:227] Negotiated authn=SASL
20:04:18.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 20:04:18.014069 (+413147us) server_negotiation.cc:366] Received TLS_HANDSHAKE NegotiatePB request
20:04:18.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 20:04:18.016187 (+ 2118us) server_negotiation.cc:378] Sending TLS_HANDSHAKE NegotiatePB response
20:04:18.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 20:04:18.115969 (+ 99782us) server_negotiation.cc:366] Received TLS_HANDSHAKE NegotiatePB request
20:04:18.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 20:04:18.116616 (+ 647us) server_negotiation.cc:663] Negotiated TLSv1.3 with cipher TLS_AES_128_GCM_SHA256 TLSv1.3 Kx=any Au=any Enc=AESGCM(128) Mac=AEAD
20:04:18.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 20:04:18.137576 (+ 20960us) negotiation.cc:326] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:45618: BlockingRecv error: failed to read from TLS socket (remote: 127.0.0.1:45618): Cannot send after transport endpoint shutdown (error 108)
20:04:18.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":14650,"thread_start_us":15290,"threads_started":1}
20:04:18.143 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1523) Notify RPC KuduRpc(method=ListTabletServers, tablet=null, attempt=1, TimeoutTracker(timeout=30000, elapsed=1039), Traces: [0ms] refreshing cache from master, [44ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.50.125:39091, [222ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.50.124:43653, [237ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.50.126:34829, [1017ms] Sub RPC ConnectToMaster: received response from server master-127.22.50.124:43653: Configuration error: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), [1017ms] Sub RPC ConnectToMaster: received response from server master-127.22.50.126:34829: Configuration error: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), [1021ms] Sub RPC ConnectToMaster: received response from server master-127.22.50.125:39091: Configuration error: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), deferred=null) after lookup exception
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.22.50.125:39091,127.22.50.124:43653,127.22.50.126:34829). Exceptions received: [org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
at java.base/java.lang.Thread.run(Thread.java:840)
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.22.50.125:39091,127.22.50.124:43653,127.22.50.126:34829). Exceptions received: [org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
at java.base/java.lang.Thread.run(Thread.java:840)
20:04:18.147 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.22.50.124:43653(127.22.50.124:43653)] handling channelInactive
20:04:18.147 [DEBUG - kudu-nio-2] (Connection.java:239) [peer master-127.22.50.126:34829(127.22.50.126:34829)] handling channelInactive
20:04:18.149 [DEBUG - kudu-nio-0] (KuduRpc.java:260) Handling a callback on RPC KuduRpc(method=ListTabletServers, tablet=null, attempt=1, TimeoutTracker(timeout=30000, elapsed=1046), Traces: [0ms] refreshing cache from master, [44ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.50.125:39091, [222ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.50.124:43653, [237ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.50.126:34829, [1017ms] Sub RPC ConnectToMaster: received response from server master-127.22.50.124:43653: Configuration error: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), [1017ms] Sub RPC ConnectToMaster: received response from server master-127.22.50.126:34829: Configuration error: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), [1021ms] Sub RPC ConnectToMaster: received response from server master-127.22.50.125:39091: Configuration error: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), deferred=null) with no deferred attached!
20:04:18.151 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.22.50.125:39091(127.22.50.125:39091)] handling channelInactive
20:04:18.152 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.22.50.125:39091(127.22.50.125:39091)] disconnecting while in state TERMINATED
20:04:18.153 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.22.50.124:43653(127.22.50.124:43653)] disconnecting while in state TERMINATED
20:04:18.154 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.22.50.126:34829(127.22.50.126:34829)] disconnecting while in state TERMINATED
20:04:18.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:04:18.153789 23861 negotiation.cc:336] Failed RPC negotiation. Trace:
20:04:18.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 20:04:17.387371 (+ 0us) reactor.cc:678] Submitting negotiation task for server connection from 127.0.0.1:56974 (local address 127.22.50.125:39091)
20:04:18.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 20:04:17.395668 (+ 8297us) server_negotiation.cc:207] Beginning negotiation
20:04:18.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 20:04:17.395698 (+ 30us) server_negotiation.cc:400] Waiting for connection header
20:04:18.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 20:04:17.502306 (+106608us) server_negotiation.cc:408] Connection header received
20:04:18.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 20:04:17.590277 (+ 87971us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
20:04:18.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 20:04:17.590310 (+ 33us) server_negotiation.cc:462] Received NEGOTIATE request from client
20:04:18.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 20:04:17.590568 (+ 258us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
20:04:18.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 20:04:17.590793 (+ 225us) server_negotiation.cc:227] Negotiated authn=SASL
20:04:18.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 20:04:17.998765 (+407972us) server_negotiation.cc:366] Received TLS_HANDSHAKE NegotiatePB request
20:04:18.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 20:04:18.003150 (+ 4385us) server_negotiation.cc:378] Sending TLS_HANDSHAKE NegotiatePB response
20:04:18.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 20:04:18.118479 (+115329us) server_negotiation.cc:366] Received TLS_HANDSHAKE NegotiatePB request
20:04:18.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 20:04:18.119149 (+ 670us) server_negotiation.cc:663] Negotiated TLSv1.3 with cipher TLS_AES_128_GCM_SHA256 TLSv1.3 Kx=any Au=any Enc=AESGCM(128) Mac=AEAD
20:04:18.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0503 20:04:18.152074 (+ 32925us) negotiation.cc:326] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:56974: BlockingRecv error: failed to read from TLS socket (remote: 127.0.0.1:56974): Cannot send after transport endpoint shutdown (error 108)
20:04:18.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":7698,"thread_start_us":7420,"threads_started":1}
20:04:18.159 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2704) Releasing all remaining resources
20:04:18.173 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
20:04:18.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:18.180032 22729 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskqmogBp/build/tsan/bin/kudu with pid 23077
20:04:18.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:18.208911 22729 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskqmogBp/build/tsan/bin/kudu with pid 23238
20:04:18.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:18.239684 22729 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskqmogBp/build/tsan/bin/kudu with pid 23385
20:04:18.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:18.267251 22729 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskqmogBp/build/tsan/bin/kudu with pid 23736
20:04:18.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:18.300971 22729 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskqmogBp/build/tsan/bin/kudu with pid 23538
20:04:18.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:18.337971 22729 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskqmogBp/build/tsan/bin/kudu with pid 23628
20:04:19.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-05-03T20:04:19Z chronyd exiting
20:04:20.631 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
20:04:20.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:20.634946 22001 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskqmogBp/build/tsan/bin/kudu with pid 22302
20:04:20.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:20.662969 22001 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskqmogBp/build/tsan/bin/kudu with pid 22442
20:04:20.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:20.689428 22001 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskqmogBp/build/tsan/bin/kudu with pid 22586
20:04:20.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:20.716435 22001 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskqmogBp/build/tsan/bin/kudu with pid 22013
20:04:20.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:04:20.745702 22186 connection.cc:570] client connection to 127.21.124.126:41563 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
20:04:20.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:04:20.746279 22186 proxy.cc:239] Call had error, refreshing address and retrying: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
20:04:20.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:20.746696 22001 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskqmogBp/build/tsan/bin/kudu with pid 22083
20:04:20.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260503 20:04:20.752144 22186 consensus_peers.cc:597] T 00000000000000000000000000000000 P f90ef18580774d77864909348b5b0693 -> Peer c87e8c6d5a524a2a84f403153fd2287b (127.21.124.126:41563): Couldn't send request to peer c87e8c6d5a524a2a84f403153fd2287b. Status: Network error: Client connection negotiation failed: client connection to 127.21.124.126:41563: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
20:04:20.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260503 20:04:20.779882 22001 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskqmogBp/build/tsan/bin/kudu with pid 22158
20:04:21.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-05-03T20:04:21Z chronyd exiting
20:04:22.996 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.22.50.125:39091,127.22.50.124:43653,127.22.50.126:34829). Exceptions received: [org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)]
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:569)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at java.base/java.lang.Thread.run(Thread.java:840)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
... 1 more
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.22.50.125:39091,127.22.50.124:43653,127.22.50.126:34829). Exceptions received: [org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)]
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:569)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at java.base/java.lang.Thread.run(Thread.java:840)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
... 1 more
20:04:23.417 [ERROR - main] (RetryRule.java:253) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: giving up after 1 attempts
20:04:23.418 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.22.50.125:39091,127.22.50.124:43653,127.22.50.126:34829). Exceptions received: [org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)]
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:569)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at java.base/java.lang.Thread.run(Thread.java:840)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
... 1 more
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.22.50.125:39091,127.22.50.124:43653,127.22.50.126:34829). Exceptions received: [org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)]
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:569)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at java.base/java.lang.Thread.run(Thread.java:840)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
... 1 more