21:21:52.448 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
21:21:52.448 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskp7EThw/test-tmp
21:21:52.448 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
21:21:52.449 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskp7EThw/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
21:21:52.450 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster5151574767910532771"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
21:21:52.762 [DEBUG - main] (MiniKuduCluster.java:195) Response:
21:21:52.762 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
21:21:52.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:52.765676 10261 mini_oidc.cc:150] Starting JWKS server
21:21:52.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:52.768929 10261 webserver.cc:492] Webserver started at http://127.0.0.1:33177/ using document root <none> and password file <none>
21:21:52.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:52.769330 10261 mini_oidc.cc:186] Starting OIDC Discovery server
21:21:52.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:52.770938 10261 webserver.cc:492] Webserver started at http://127.0.0.1:42245/ using document root <none> and password file <none>
21:21:52.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-01-28T21:21:52Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
21:21:52.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-01-28T21:21:52Z Disabled control of system clock
21:21:52.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:52.826364 10261 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskp7EThw/build/tsan/bin/kudu
21:21:52.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskp7EThw/build/tsan/bin/kudu
21:21:52.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster5151574767910532771/master-0/wal
21:21:52.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster5151574767910532771/master-0/data
21:21:52.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:21:52.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:21:52.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:21:52.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:21:52.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:21:52.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:21:52.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster5151574767910532771/master-0/logs
21:21:52.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster5151574767910532771/master-0/data/info.pb
21:21:52.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:21:52.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:21:52.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:21:52.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:21:52.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:21:52.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:21:52.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:21:52.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:21:52.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:21:52.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:21:52.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:21:52.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:21:52.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.5.126:42447
21:21:52.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.5.126
21:21:52.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:21:52.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.5.84:40825
21:21:52.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:21:52.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:21:52.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:21:52.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:21:52.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.5.126:42447,127.10.5.125:35615,127.10.5.124:43343
21:21:52.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:21:52.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33177/jwks.json with env {}
21:21:53.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:21:53.115726 10273 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:21:53.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:21:53.116309 10273 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:21:53.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:21:53.116814 10273 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:21:53.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:21:53.148088 10273 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:21:53.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:21:53.148376 10273 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:21:53.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:21:53.148574 10273 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:21:53.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:21:53.148766 10273 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:21:53.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:21:53.148957 10273 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:21:53.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:21:53.149104 10273 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33177/jwks.json
21:21:53.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:53.184684 10273 master_runner.cc:386] Master server non-default flags:
21:21:53.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:21:53.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.5.84:40825
21:21:53.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:21:53.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:21:53.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster5151574767910532771/master-0/data
21:21:53.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster5151574767910532771/master-0/wal
21:21:53.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:21:53.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.5.126:42447,127.10.5.125:35615,127.10.5.124:43343
21:21:53.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:21:53.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:21:53.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:21:53.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.5.126:42447
21:21:53.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:21:53.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:21:53.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:21:53.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33177/jwks.json
21:21:53.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:21:53.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:21:53.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster5151574767910532771/master-0/data/info.pb
21:21:53.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.5.126
21:21:53.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:21:53.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:21:53.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:21:53.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:21:53.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:21:53.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:21:53.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster5151574767910532771/master-0/logs
21:21:53.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:21:53.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:21:53.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:21:53.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:21:53.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:21:53.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3722ed49c19ce54c7feae2fa944e32b1d4ed8d2d
21:21:53.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:21:53.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 28 Jan 2026 21:02:00 UTC on 24a791456cd2
21:21:53.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10237
21:21:53.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:21:53.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:53.185947 10273 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:21:53.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:53.187495 10273 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:21:53.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:21:53.197965 10279 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:21:54.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:21:54.600937 10278 debug-util.cc:398] Leaking SignalData structure 0x7b080002ec80 after lost signal to thread 10273
21:21:54.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:21:54.736649 10273 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.539s user 0.546s sys 0.992s
21:21:54.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:21:54.736963 10273 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.539s user 0.546s sys 0.992s
21:21:54.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:21:53.198244 10280 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:21:54.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:21:54.738840 10282 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:21:54.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:21:54.741209 10281 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1538 milliseconds
21:21:54.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:54.741325 10273 server_base.cc:1055] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:21:54.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:54.742440 10273 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:21:54.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:54.744894 10273 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:21:54.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:54.746236 10273 hybrid_clock.cc:648] HybridClock initialized: now 1769635314746188 us; error 50 us; skew 500 ppm
21:21:54.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:54.747001 10273 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:21:54.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:54.752825 10273 webserver.cc:492] Webserver started at http://127.10.5.126:43107/ using document root <none> and password file <none>
21:21:54.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:54.753719 10273 fs_manager.cc:362] Metadata directory not provided
21:21:54.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:54.753917 10273 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:21:54.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:54.754328 10273 server_base.cc:908] This appears to be a new deployment of Kudu; creating new FS layout
21:21:54.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:54.758473 10273 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster5151574767910532771/master-0/data/instance:
21:21:54.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cf87ba5a13b14d2dbd2901abf3152ee0"
21:21:54.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-01-28 21:21:54 on dist-test-slave-hq3s"
21:21:54.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:54.759514 10273 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster5151574767910532771/master-0/wal/instance:
21:21:54.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cf87ba5a13b14d2dbd2901abf3152ee0"
21:21:54.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-01-28 21:21:54 on dist-test-slave-hq3s"
21:21:54.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:54.766410 10273 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.001s sys 0.005s
21:21:54.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:54.771946 10289 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:21:54.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:54.773099 10273 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.001s
21:21:54.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:54.773409 10273 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster5151574767910532771/master-0/data,/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster5151574767910532771/master-0/wal
21:21:54.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cf87ba5a13b14d2dbd2901abf3152ee0"
21:21:54.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-01-28 21:21:54 on dist-test-slave-hq3s"
21:21:54.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:54.773729 10273 fs_report.cc:389] FS layout report
21:21:54.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:21:54.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster5151574767910532771/master-0/wal
21:21:54.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster5151574767910532771/master-0/wal
21:21:54.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster5151574767910532771/master-0/data/data
21:21:54.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:21:54.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:21:54.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:21:54.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:21:54.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:21:54.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:21:54.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:21:54.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:21:54.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:21:54.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:21:54.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:21:54.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:54.816342 10273 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:21:54.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:54.817755 10273 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:21:54.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:54.818148 10273 kserver.cc:163] Server-wide thread pool size limit: 3276
21:21:54.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:54.884606 10273 rpc_server.cc:307] RPC server started. Bound to: 127.10.5.126:42447
21:21:54.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:54.884649 10340 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.10.5.126:42447 every 8 connection(s)
21:21:54.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:54.887228 10273 server_base.cc:1192] Dumped server information to /tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster5151574767910532771/master-0/data/info.pb
21:21:54.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:54.892757 10341 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
21:21:54.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:54.896726 10261 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskp7EThw/build/tsan/bin/kudu as pid 10273
21:21:54.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:54.897092 10261 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster5151574767910532771/master-0/wal/instance
21:21:54.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:54.900293 10261 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskp7EThw/build/tsan/bin/kudu
21:21:54.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskp7EThw/build/tsan/bin/kudu
21:21:54.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster5151574767910532771/master-1/wal
21:21:54.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster5151574767910532771/master-1/data
21:21:54.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:21:54.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:21:54.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:21:54.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:21:54.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:21:54.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:21:54.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster5151574767910532771/master-1/logs
21:21:54.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster5151574767910532771/master-1/data/info.pb
21:21:54.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:21:54.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:21:54.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:21:54.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:21:54.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:21:54.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:21:54.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:21:54.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:21:54.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:21:54.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:21:54.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:21:54.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:21:54.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.5.125:35615
21:21:54.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.5.125
21:21:54.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:21:54.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.5.84:40825
21:21:54.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:21:54.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:21:54.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:21:54.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:21:54.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.5.126:42447,127.10.5.125:35615,127.10.5.124:43343
21:21:54.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:21:54.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33177/jwks.json with env {}
21:21:54.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:54.903740 10341 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.5.126" port: 42447 } has no permanent_uuid. Determining permanent_uuid...
21:21:54.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:54.933817 10341 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.5.125" port: 35615 } has no permanent_uuid. Determining permanent_uuid...
21:21:54.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:21:54.935916 10291 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.10.5.125:35615: connect: Connection refused (error 111)
21:21:54.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:21:54.939738 10341 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.10.5.125:35615: Network error: Client connection negotiation failed: client connection to 127.10.5.125:35615: connect: Connection refused (error 111)
21:21:54.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:54.989300 10341 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.5.125" port: 35615 } attempt: 1
21:21:54.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:21:54.993186 10341 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.10.5.125:35615: Network error: Client connection negotiation failed: client connection to 127.10.5.125:35615: connect: Connection refused (error 111)
21:21:55.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:55.061643 10341 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.5.125" port: 35615 } attempt: 2
21:21:55.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:21:55.065392 10341 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.10.5.125:35615: Network error: Client connection negotiation failed: client connection to 127.10.5.125:35615: connect: Connection refused (error 111)
21:21:55.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:55.156839 10341 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.5.125" port: 35615 } attempt: 3
21:21:55.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:21:55.160995 10341 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.10.5.125:35615: Network error: Client connection negotiation failed: client connection to 127.10.5.125:35615: connect: Connection refused (error 111)
21:21:55.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:21:55.195726 10343 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:21:55.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:21:55.196285 10343 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:21:55.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:21:55.196775 10343 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:21:55.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:21:55.228538 10343 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:21:55.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:21:55.228845 10343 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:21:55.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:21:55.229045 10343 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:21:55.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:21:55.229238 10343 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:21:55.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:21:55.229430 10343 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:21:55.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:21:55.229583 10343 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33177/jwks.json
21:21:55.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:55.265583 10343 master_runner.cc:386] Master server non-default flags:
21:21:55.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:21:55.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.5.84:40825
21:21:55.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:21:55.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:21:55.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster5151574767910532771/master-1/data
21:21:55.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster5151574767910532771/master-1/wal
21:21:55.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:21:55.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.5.126:42447,127.10.5.125:35615,127.10.5.124:43343
21:21:55.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:21:55.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:21:55.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:21:55.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.5.125:35615
21:21:55.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:21:55.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:21:55.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:21:55.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33177/jwks.json
21:21:55.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:21:55.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:21:55.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster5151574767910532771/master-1/data/info.pb
21:21:55.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.5.125
21:21:55.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:21:55.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:21:55.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:21:55.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:21:55.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:21:55.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:21:55.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster5151574767910532771/master-1/logs
21:21:55.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:21:55.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:21:55.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:21:55.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:21:55.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:21:55.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3722ed49c19ce54c7feae2fa944e32b1d4ed8d2d
21:21:55.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:21:55.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 28 Jan 2026 21:02:00 UTC on 24a791456cd2
21:21:55.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10237
21:21:55.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:21:55.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:55.266850 10343 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:21:55.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:55.268453 10343 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:21:55.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:21:55.278687 10352 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:21:55.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:55.304467 10341 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.5.125" port: 35615 } attempt: 4
21:21:55.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:21:55.310858 10341 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.10.5.125:35615: Network error: Client connection negotiation failed: client connection to 127.10.5.125:35615: connect: Connection refused (error 111)
21:21:55.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:55.610335 10341 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.5.125" port: 35615 } attempt: 5
21:21:55.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:21:55.617834 10341 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.10.5.125:35615: Network error: Client connection negotiation failed: client connection to 127.10.5.125:35615: connect: Connection refused (error 111)
21:21:56.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:56.165289 10341 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.5.125" port: 35615 } attempt: 6
21:21:56.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:21:56.198951 10341 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.10.5.125:35615: Network error: Client connection negotiation failed: client connection to 127.10.5.125:35615: connect: Connection refused (error 111)
21:21:56.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:21:56.682145 10351 debug-util.cc:398] Leaking SignalData structure 0x7b080002ec80 after lost signal to thread 10343
21:21:56.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:21:55.280014 10353 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:21:56.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:21:56.813267 10351 kernel_stack_watchdog.cc:198] Thread 10343 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 400ms:
21:21:56.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
21:21:56.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
21:21:56.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
21:21:56.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
21:21:56.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:21:56.814213 10343 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.535s user 0.002s sys 0.000s
21:21:56.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:21:56.815064 10343 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.536s user 0.002s sys 0.000s
21:21:56.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:21:56.816056 10355 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:21:56.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:21:56.818327 10354 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1533 milliseconds
21:21:56.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:56.818332 10343 server_base.cc:1055] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:21:56.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:56.819528 10343 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:21:56.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:56.821830 10343 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:21:56.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:56.823171 10343 hybrid_clock.cc:648] HybridClock initialized: now 1769635316823128 us; error 40 us; skew 500 ppm
21:21:56.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:56.824005 10343 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:21:56.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:56.829973 10343 webserver.cc:492] Webserver started at http://127.10.5.125:41027/ using document root <none> and password file <none>
21:21:56.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:56.830855 10343 fs_manager.cc:362] Metadata directory not provided
21:21:56.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:56.831064 10343 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:21:56.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:56.831516 10343 server_base.cc:908] This appears to be a new deployment of Kudu; creating new FS layout
21:21:56.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:56.835795 10343 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster5151574767910532771/master-1/data/instance:
21:21:56.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c789c3f650204bf4893740b2759914c9"
21:21:56.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-01-28 21:21:56 on dist-test-slave-hq3s"
21:21:56.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:56.837034 10343 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster5151574767910532771/master-1/wal/instance:
21:21:56.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c789c3f650204bf4893740b2759914c9"
21:21:56.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-01-28 21:21:56 on dist-test-slave-hq3s"
21:21:56.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:56.843904 10343 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.004s sys 0.004s
21:21:56.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:56.849321 10364 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:21:56.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:56.850315 10343 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.005s sys 0.000s
21:21:56.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:56.850662 10343 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster5151574767910532771/master-1/data,/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster5151574767910532771/master-1/wal
21:21:56.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c789c3f650204bf4893740b2759914c9"
21:21:56.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-01-28 21:21:56 on dist-test-slave-hq3s"
21:21:56.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:56.850982 10343 fs_report.cc:389] FS layout report
21:21:56.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:21:56.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster5151574767910532771/master-1/wal
21:21:56.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster5151574767910532771/master-1/wal
21:21:56.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster5151574767910532771/master-1/data/data
21:21:56.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:21:56.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:21:56.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:21:56.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:21:56.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:21:56.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:21:56.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:21:56.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:21:56.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:21:56.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:21:56.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:21:56.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:56.909813 10343 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:21:56.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:56.911280 10343 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:21:56.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:56.911756 10343 kserver.cc:163] Server-wide thread pool size limit: 3276
21:21:56.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:56.979022 10343 rpc_server.cc:307] RPC server started. Bound to: 127.10.5.125:35615
21:21:56.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:56.979094 10415 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.10.5.125:35615 every 8 connection(s)
21:21:56.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:56.981685 10343 server_base.cc:1192] Dumped server information to /tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster5151574767910532771/master-1/data/info.pb
21:21:56.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:56.987249 10416 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
21:21:56.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:56.992214 10261 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskp7EThw/build/tsan/bin/kudu as pid 10343
21:21:56.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:56.992714 10261 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster5151574767910532771/master-1/wal/instance
21:21:56.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:56.995576 10261 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskp7EThw/build/tsan/bin/kudu
21:21:56.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskp7EThw/build/tsan/bin/kudu
21:21:56.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster5151574767910532771/master-2/wal
21:21:56.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster5151574767910532771/master-2/data
21:21:56.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:21:56.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:21:56.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:21:56.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:21:56.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:21:56.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:21:56.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster5151574767910532771/master-2/logs
21:21:56.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster5151574767910532771/master-2/data/info.pb
21:21:56.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:21:56.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:21:56.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:21:56.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:21:56.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:21:56.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:21:56.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:21:56.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:21:56.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:21:56.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:21:56.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:21:56.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:21:56.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.5.124:43343
21:21:56.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.5.124
21:21:56.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:21:56.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.5.84:40825
21:21:56.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:21:56.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:21:56.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:21:56.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:21:56.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.5.126:42447,127.10.5.125:35615,127.10.5.124:43343
21:21:56.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:21:56.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33177/jwks.json with env {}
21:21:57.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:57.000322 10416 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.5.126" port: 42447 } has no permanent_uuid. Determining permanent_uuid...
21:21:57.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:57.021098 10416 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.5.125" port: 35615 } has no permanent_uuid. Determining permanent_uuid...
21:21:57.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:57.033703 10416 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.5.124" port: 43343 } has no permanent_uuid. Determining permanent_uuid...
21:21:57.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:21:57.035809 10368 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.10.5.124:43343: connect: Connection refused (error 111)
21:21:57.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:21:57.039736 10416 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.10.5.124:43343: Network error: Client connection negotiation failed: client connection to 127.10.5.124:43343: connect: Connection refused (error 111)
21:21:57.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:57.089156 10416 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.5.124" port: 43343 } attempt: 1
21:21:57.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:21:57.092893 10416 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.10.5.124:43343: Network error: Client connection negotiation failed: client connection to 127.10.5.124:43343: connect: Connection refused (error 111)
21:21:57.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:57.161352 10416 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.5.124" port: 43343 } attempt: 2
21:21:57.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:21:57.165078 10416 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.10.5.124:43343: Network error: Client connection negotiation failed: client connection to 127.10.5.124:43343: connect: Connection refused (error 111)
21:21:57.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:57.256532 10416 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.5.124" port: 43343 } attempt: 3
21:21:57.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:57.259436 10341 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.5.125" port: 35615 } attempt: 7
21:21:57.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:21:57.261132 10416 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.10.5.124:43343: Network error: Client connection negotiation failed: client connection to 127.10.5.124:43343: connect: Connection refused (error 111)
21:21:57.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:57.271004 10341 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.5.124" port: 43343 } has no permanent_uuid. Determining permanent_uuid...
21:21:57.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:21:57.276382 10341 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.10.5.124:43343: Network error: Client connection negotiation failed: client connection to 127.10.5.124:43343: connect: Connection refused (error 111)
21:21:57.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:21:57.298521 10418 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:21:57.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:21:57.299073 10418 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:21:57.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:21:57.299624 10418 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:21:57.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:21:57.331205 10418 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:21:57.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:21:57.331568 10418 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:21:57.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:21:57.331902 10418 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:21:57.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:21:57.332180 10418 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:21:57.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:21:57.332466 10418 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:21:57.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:21:57.332682 10418 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33177/jwks.json
21:21:57.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:57.334753 10341 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.5.124" port: 43343 } attempt: 1
21:21:57.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:21:57.338505 10341 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.10.5.124:43343: Network error: Client connection negotiation failed: client connection to 127.10.5.124:43343: connect: Connection refused (error 111)
21:21:57.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:57.369395 10418 master_runner.cc:386] Master server non-default flags:
21:21:57.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:21:57.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.5.84:40825
21:21:57.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:21:57.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:21:57.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster5151574767910532771/master-2/data
21:21:57.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster5151574767910532771/master-2/wal
21:21:57.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:21:57.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.5.126:42447,127.10.5.125:35615,127.10.5.124:43343
21:21:57.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:21:57.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:21:57.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:21:57.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.5.124:43343
21:21:57.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:21:57.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:21:57.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:21:57.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33177/jwks.json
21:21:57.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:21:57.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:21:57.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster5151574767910532771/master-2/data/info.pb
21:21:57.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.5.124
21:21:57.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:21:57.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:21:57.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:21:57.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:21:57.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:21:57.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:21:57.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster5151574767910532771/master-2/logs
21:21:57.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:21:57.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:21:57.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:21:57.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:21:57.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:21:57.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3722ed49c19ce54c7feae2fa944e32b1d4ed8d2d
21:21:57.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:21:57.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 28 Jan 2026 21:02:00 UTC on 24a791456cd2
21:21:57.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10237
21:21:57.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:21:57.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:57.371017 10418 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:21:57.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:57.373188 10418 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:21:57.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:21:57.384836 10430 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:21:57.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:57.404563 10416 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.5.124" port: 43343 } attempt: 4
21:21:57.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:21:57.408233 10416 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.10.5.124:43343: Network error: Client connection negotiation failed: client connection to 127.10.5.124:43343: connect: Connection refused (error 111)
21:21:57.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:57.419847 10341 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.5.124" port: 43343 } attempt: 2
21:21:57.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:21:57.423298 10341 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.10.5.124:43343: Network error: Client connection negotiation failed: client connection to 127.10.5.124:43343: connect: Connection refused (error 111)
21:21:57.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:57.508682 10341 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.5.124" port: 43343 } attempt: 3
21:21:57.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:21:57.513589 10341 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.10.5.124:43343: Network error: Client connection negotiation failed: client connection to 127.10.5.124:43343: connect: Connection refused (error 111)
21:21:57.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:57.654064 10341 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.5.124" port: 43343 } attempt: 4
21:21:57.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:21:57.659222 10341 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.10.5.124:43343: Network error: Client connection negotiation failed: client connection to 127.10.5.124:43343: connect: Connection refused (error 111)
21:21:57.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:57.707648 10416 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.5.124" port: 43343 } attempt: 5
21:21:57.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:21:57.712553 10416 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.10.5.124:43343: Network error: Client connection negotiation failed: client connection to 127.10.5.124:43343: connect: Connection refused (error 111)
21:21:57.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:57.942854 10341 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.5.124" port: 43343 } attempt: 5
21:21:57.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:21:57.948196 10341 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.10.5.124:43343: Network error: Client connection negotiation failed: client connection to 127.10.5.124:43343: connect: Connection refused (error 111)
21:21:58.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:58.260099 10416 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.5.124" port: 43343 } attempt: 6
21:21:58.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:21:58.268620 10416 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.10.5.124:43343: Network error: Client connection negotiation failed: client connection to 127.10.5.124:43343: connect: Connection refused (error 111)
21:21:58.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:58.500705 10341 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.5.124" port: 43343 } attempt: 6
21:21:58.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:21:58.508770 10341 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.10.5.124:43343: Network error: Client connection negotiation failed: client connection to 127.10.5.124:43343: connect: Connection refused (error 111)
21:21:58.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:21:57.389151 10433 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:21:58.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:21:57.386054 10431 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:21:58.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:21:58.538261 10432 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1149 milliseconds
21:21:58.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:58.538375 10418 server_base.cc:1055] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:21:58.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:58.539577 10418 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:21:58.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:58.542433 10418 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:21:58.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:58.543857 10418 hybrid_clock.cc:648] HybridClock initialized: now 1769635318543813 us; error 54 us; skew 500 ppm
21:21:58.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:58.544656 10418 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:21:58.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:58.551172 10418 webserver.cc:492] Webserver started at http://127.10.5.124:41527/ using document root <none> and password file <none>
21:21:58.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:58.552073 10418 fs_manager.cc:362] Metadata directory not provided
21:21:58.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:58.552258 10418 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:21:58.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:58.552654 10418 server_base.cc:908] This appears to be a new deployment of Kudu; creating new FS layout
21:21:58.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:58.557070 10418 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster5151574767910532771/master-2/data/instance:
21:21:58.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e2d8a542014049bf87a3c845a5ca22e6"
21:21:58.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-01-28 21:21:58 on dist-test-slave-hq3s"
21:21:58.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:58.558075 10418 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster5151574767910532771/master-2/wal/instance:
21:21:58.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e2d8a542014049bf87a3c845a5ca22e6"
21:21:58.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-01-28 21:21:58 on dist-test-slave-hq3s"
21:21:58.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:58.564762 10418 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.007s sys 0.001s
21:21:58.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:58.570066 10444 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:21:58.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:58.571060 10418 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.002s sys 0.000s
21:21:58.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:58.571341 10418 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster5151574767910532771/master-2/data,/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster5151574767910532771/master-2/wal
21:21:58.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e2d8a542014049bf87a3c845a5ca22e6"
21:21:58.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-01-28 21:21:58 on dist-test-slave-hq3s"
21:21:58.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:58.571653 10418 fs_report.cc:389] FS layout report
21:21:58.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:21:58.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster5151574767910532771/master-2/wal
21:21:58.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster5151574767910532771/master-2/wal
21:21:58.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster5151574767910532771/master-2/data/data
21:21:58.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:21:58.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:21:58.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:21:58.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:21:58.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:21:58.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:21:58.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:21:58.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:21:58.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:21:58.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:21:58.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:21:58.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:58.628306 10418 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:21:58.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:58.629726 10418 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:21:58.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:58.630177 10418 kserver.cc:163] Server-wide thread pool size limit: 3276
21:21:58.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:58.697710 10418 rpc_server.cc:307] RPC server started. Bound to: 127.10.5.124:43343
21:21:58.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:58.697765 10495 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.10.5.124:43343 every 8 connection(s)
21:21:58.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:58.700405 10418 server_base.cc:1192] Dumped server information to /tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster5151574767910532771/master-2/data/info.pb
21:21:58.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:58.706410 10496 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
21:21:58.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:58.707273 10261 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskp7EThw/build/tsan/bin/kudu as pid 10418
21:21:58.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:58.707638 10261 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster5151574767910532771/master-2/wal/instance
21:21:58.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:58.719558 10496 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.5.126" port: 42447 } has no permanent_uuid. Determining permanent_uuid...
21:21:58.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:58.749198 10496 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.5.125" port: 35615 } has no permanent_uuid. Determining permanent_uuid...
21:21:58.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:58.758065 10496 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.5.124" port: 43343 } has no permanent_uuid. Determining permanent_uuid...
21:21:58.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:58.773243 10496 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P e2d8a542014049bf87a3c845a5ca22e6: Bootstrap starting.
21:21:58.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:58.779801 10496 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P e2d8a542014049bf87a3c845a5ca22e6: Neither blocks nor log segments found. Creating new log.
21:21:58.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:58.781793 10496 log.cc:826] T 00000000000000000000000000000000 P e2d8a542014049bf87a3c845a5ca22e6: Log is configured to *not* fsync() on all Append() calls
21:21:58.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:58.786635 10496 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P e2d8a542014049bf87a3c845a5ca22e6: No bootstrap required, opened a new log
21:21:58.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:58.805347 10496 raft_consensus.cc:359] T 00000000000000000000000000000000 P e2d8a542014049bf87a3c845a5ca22e6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cf87ba5a13b14d2dbd2901abf3152ee0" member_type: VOTER last_known_addr { host: "127.10.5.126" port: 42447 } } peers { permanent_uuid: "c789c3f650204bf4893740b2759914c9" member_type: VOTER last_known_addr { host: "127.10.5.125" port: 35615 } } peers { permanent_uuid: "e2d8a542014049bf87a3c845a5ca22e6" member_type: VOTER last_known_addr { host: "127.10.5.124" port: 43343 } }
21:21:58.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:58.805984 10496 raft_consensus.cc:385] T 00000000000000000000000000000000 P e2d8a542014049bf87a3c845a5ca22e6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:21:58.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:58.806231 10496 raft_consensus.cc:740] T 00000000000000000000000000000000 P e2d8a542014049bf87a3c845a5ca22e6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e2d8a542014049bf87a3c845a5ca22e6, State: Initialized, Role: FOLLOWER
21:21:58.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:58.806912 10496 consensus_queue.cc:260] T 00000000000000000000000000000000 P e2d8a542014049bf87a3c845a5ca22e6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cf87ba5a13b14d2dbd2901abf3152ee0" member_type: VOTER last_known_addr { host: "127.10.5.126" port: 42447 } } peers { permanent_uuid: "c789c3f650204bf4893740b2759914c9" member_type: VOTER last_known_addr { host: "127.10.5.125" port: 35615 } } peers { permanent_uuid: "e2d8a542014049bf87a3c845a5ca22e6" member_type: VOTER last_known_addr { host: "127.10.5.124" port: 43343 } }
21:21:58.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:58.809110 10505 sys_catalog.cc:455] T 00000000000000000000000000000000 P e2d8a542014049bf87a3c845a5ca22e6 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cf87ba5a13b14d2dbd2901abf3152ee0" member_type: VOTER last_known_addr { host: "127.10.5.126" port: 42447 } } peers { permanent_uuid: "c789c3f650204bf4893740b2759914c9" member_type: VOTER last_known_addr { host: "127.10.5.125" port: 35615 } } peers { permanent_uuid: "e2d8a542014049bf87a3c845a5ca22e6" member_type: VOTER last_known_addr { host: "127.10.5.124" port: 43343 } } }
21:21:58.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:58.809698 10505 sys_catalog.cc:458] T 00000000000000000000000000000000 P e2d8a542014049bf87a3c845a5ca22e6 [sys.catalog]: This master's current role is: FOLLOWER
21:21:58.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:58.810595 10496 sys_catalog.cc:565] T 00000000000000000000000000000000 P e2d8a542014049bf87a3c845a5ca22e6 [sys.catalog]: configured and running, proceeding with master startup.
21:21:58.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:21:58.836650 10516 catalog_manager.cc:1568] T 00000000000000000000000000000000 P e2d8a542014049bf87a3c845a5ca22e6: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:21:58.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:21:58.836980 10516 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:21:58.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:58.830037 10418 master_runner.cc:186] Error getting master registration for 127.10.5.126:42447: OK, instance_id { permanent_uuid: "cf87ba5a13b14d2dbd2901abf3152ee0" instance_seqno: 1769635314867051 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
21:21:58.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:58.850461 10418 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
21:21:58.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:21:58.885238 10450 tablet.cc:2378] T 00000000000000000000000000000000 P e2d8a542014049bf87a3c845a5ca22e6: Can't schedule compaction. Clean time has not been advanced past its initial value.
21:21:59.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:59.157848 10505 raft_consensus.cc:493] T 00000000000000000000000000000000 P e2d8a542014049bf87a3c845a5ca22e6 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:21:59.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:59.158391 10505 raft_consensus.cc:515] T 00000000000000000000000000000000 P e2d8a542014049bf87a3c845a5ca22e6 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cf87ba5a13b14d2dbd2901abf3152ee0" member_type: VOTER last_known_addr { host: "127.10.5.126" port: 42447 } } peers { permanent_uuid: "c789c3f650204bf4893740b2759914c9" member_type: VOTER last_known_addr { host: "127.10.5.125" port: 35615 } } peers { permanent_uuid: "e2d8a542014049bf87a3c845a5ca22e6" member_type: VOTER last_known_addr { host: "127.10.5.124" port: 43343 } }
21:21:59.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:59.160931 10505 leader_election.cc:290] T 00000000000000000000000000000000 P e2d8a542014049bf87a3c845a5ca22e6 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers cf87ba5a13b14d2dbd2901abf3152ee0 (127.10.5.126:42447), c789c3f650204bf4893740b2759914c9 (127.10.5.125:35615)
21:21:59.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:59.329099 10416 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.5.124" port: 43343 } attempt: 7
21:21:59.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:59.353765 10416 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c789c3f650204bf4893740b2759914c9: Bootstrap starting.
21:21:59.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:59.362134 10416 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P c789c3f650204bf4893740b2759914c9: Neither blocks nor log segments found. Creating new log.
21:21:59.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:59.364476 10416 log.cc:826] T 00000000000000000000000000000000 P c789c3f650204bf4893740b2759914c9: Log is configured to *not* fsync() on all Append() calls
21:21:59.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:59.369598 10416 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c789c3f650204bf4893740b2759914c9: No bootstrap required, opened a new log
21:21:59.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:59.397974 10416 raft_consensus.cc:359] T 00000000000000000000000000000000 P c789c3f650204bf4893740b2759914c9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cf87ba5a13b14d2dbd2901abf3152ee0" member_type: VOTER last_known_addr { host: "127.10.5.126" port: 42447 } } peers { permanent_uuid: "c789c3f650204bf4893740b2759914c9" member_type: VOTER last_known_addr { host: "127.10.5.125" port: 35615 } } peers { permanent_uuid: "e2d8a542014049bf87a3c845a5ca22e6" member_type: VOTER last_known_addr { host: "127.10.5.124" port: 43343 } }
21:21:59.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:59.399042 10416 raft_consensus.cc:385] T 00000000000000000000000000000000 P c789c3f650204bf4893740b2759914c9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:21:59.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:59.399361 10416 raft_consensus.cc:740] T 00000000000000000000000000000000 P c789c3f650204bf4893740b2759914c9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c789c3f650204bf4893740b2759914c9, State: Initialized, Role: FOLLOWER
21:21:59.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:59.400297 10416 consensus_queue.cc:260] T 00000000000000000000000000000000 P c789c3f650204bf4893740b2759914c9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cf87ba5a13b14d2dbd2901abf3152ee0" member_type: VOTER last_known_addr { host: "127.10.5.126" port: 42447 } } peers { permanent_uuid: "c789c3f650204bf4893740b2759914c9" member_type: VOTER last_known_addr { host: "127.10.5.125" port: 35615 } } peers { permanent_uuid: "e2d8a542014049bf87a3c845a5ca22e6" member_type: VOTER last_known_addr { host: "127.10.5.124" port: 43343 } }
21:21:59.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:59.405352 10416 sys_catalog.cc:565] T 00000000000000000000000000000000 P c789c3f650204bf4893740b2759914c9 [sys.catalog]: configured and running, proceeding with master startup.
21:21:59.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:59.405747 10391 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "e2d8a542014049bf87a3c845a5ca22e6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c789c3f650204bf4893740b2759914c9" is_pre_election: true
21:21:59.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:59.407531 10391 raft_consensus.cc:2468] T 00000000000000000000000000000000 P c789c3f650204bf4893740b2759914c9 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate e2d8a542014049bf87a3c845a5ca22e6 in term 0.
21:21:59.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:59.409273 10521 sys_catalog.cc:455] T 00000000000000000000000000000000 P c789c3f650204bf4893740b2759914c9 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cf87ba5a13b14d2dbd2901abf3152ee0" member_type: VOTER last_known_addr { host: "127.10.5.126" port: 42447 } } peers { permanent_uuid: "c789c3f650204bf4893740b2759914c9" member_type: VOTER last_known_addr { host: "127.10.5.125" port: 35615 } } peers { permanent_uuid: "e2d8a542014049bf87a3c845a5ca22e6" member_type: VOTER last_known_addr { host: "127.10.5.124" port: 43343 } } }
21:21:59.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:59.409751 10446 leader_election.cc:304] T 00000000000000000000000000000000 P e2d8a542014049bf87a3c845a5ca22e6 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: c789c3f650204bf4893740b2759914c9, e2d8a542014049bf87a3c845a5ca22e6; no voters:
21:21:59.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:59.410634 10505 raft_consensus.cc:2804] T 00000000000000000000000000000000 P e2d8a542014049bf87a3c845a5ca22e6 [term 0 FOLLOWER]: Leader pre-election won for term 1
21:21:59.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:59.411000 10505 raft_consensus.cc:493] T 00000000000000000000000000000000 P e2d8a542014049bf87a3c845a5ca22e6 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
21:21:59.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:59.410871 10521 sys_catalog.cc:458] T 00000000000000000000000000000000 P c789c3f650204bf4893740b2759914c9 [sys.catalog]: This master's current role is: FOLLOWER
21:21:59.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:59.411324 10505 raft_consensus.cc:3060] T 00000000000000000000000000000000 P e2d8a542014049bf87a3c845a5ca22e6 [term 0 FOLLOWER]: Advancing to term 1
21:21:59.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:59.412214 10521 raft_consensus.cc:493] T 00000000000000000000000000000000 P c789c3f650204bf4893740b2759914c9 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:21:59.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:59.413825 10521 raft_consensus.cc:515] T 00000000000000000000000000000000 P c789c3f650204bf4893740b2759914c9 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cf87ba5a13b14d2dbd2901abf3152ee0" member_type: VOTER last_known_addr { host: "127.10.5.126" port: 42447 } } peers { permanent_uuid: "c789c3f650204bf4893740b2759914c9" member_type: VOTER last_known_addr { host: "127.10.5.125" port: 35615 } } peers { permanent_uuid: "e2d8a542014049bf87a3c845a5ca22e6" member_type: VOTER last_known_addr { host: "127.10.5.124" port: 43343 } }
21:21:59.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:21:59.418951 10370 tablet.cc:2378] T 00000000000000000000000000000000 P c789c3f650204bf4893740b2759914c9: Can't schedule compaction. Clean time has not been advanced past its initial value.
21:21:59.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:59.421716 10521 leader_election.cc:290] T 00000000000000000000000000000000 P c789c3f650204bf4893740b2759914c9 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers cf87ba5a13b14d2dbd2901abf3152ee0 (127.10.5.126:42447), e2d8a542014049bf87a3c845a5ca22e6 (127.10.5.124:43343)
21:21:59.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:59.421795 10471 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "c789c3f650204bf4893740b2759914c9" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e2d8a542014049bf87a3c845a5ca22e6" is_pre_election: true
21:21:59.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:59.424057 10505 raft_consensus.cc:515] T 00000000000000000000000000000000 P e2d8a542014049bf87a3c845a5ca22e6 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cf87ba5a13b14d2dbd2901abf3152ee0" member_type: VOTER last_known_addr { host: "127.10.5.126" port: 42447 } } peers { permanent_uuid: "c789c3f650204bf4893740b2759914c9" member_type: VOTER last_known_addr { host: "127.10.5.125" port: 35615 } } peers { permanent_uuid: "e2d8a542014049bf87a3c845a5ca22e6" member_type: VOTER last_known_addr { host: "127.10.5.124" port: 43343 } }
21:21:59.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:59.426110 10471 raft_consensus.cc:2393] T 00000000000000000000000000000000 P e2d8a542014049bf87a3c845a5ca22e6 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate c789c3f650204bf4893740b2759914c9 in current term 1: Already voted for candidate e2d8a542014049bf87a3c845a5ca22e6 in this term.
21:21:59.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:59.438510 10505 leader_election.cc:290] T 00000000000000000000000000000000 P e2d8a542014049bf87a3c845a5ca22e6 [CANDIDATE]: Term 1 election: Requested vote from peers cf87ba5a13b14d2dbd2901abf3152ee0 (127.10.5.126:42447), c789c3f650204bf4893740b2759914c9 (127.10.5.125:35615)
21:21:59.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:59.439486 10391 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "e2d8a542014049bf87a3c845a5ca22e6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c789c3f650204bf4893740b2759914c9"
21:21:59.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:59.440068 10391 raft_consensus.cc:3060] T 00000000000000000000000000000000 P c789c3f650204bf4893740b2759914c9 [term 0 FOLLOWER]: Advancing to term 1
21:21:59.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:59.446770 10391 raft_consensus.cc:2468] T 00000000000000000000000000000000 P c789c3f650204bf4893740b2759914c9 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e2d8a542014049bf87a3c845a5ca22e6 in term 1.
21:21:59.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:59.447858 10446 leader_election.cc:304] T 00000000000000000000000000000000 P e2d8a542014049bf87a3c845a5ca22e6 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: c789c3f650204bf4893740b2759914c9, e2d8a542014049bf87a3c845a5ca22e6; no voters:
21:21:59.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:59.448643 10505 raft_consensus.cc:2804] T 00000000000000000000000000000000 P e2d8a542014049bf87a3c845a5ca22e6 [term 1 FOLLOWER]: Leader election won for term 1
21:21:59.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:59.450562 10505 raft_consensus.cc:697] T 00000000000000000000000000000000 P e2d8a542014049bf87a3c845a5ca22e6 [term 1 LEADER]: Becoming Leader. State: Replica: e2d8a542014049bf87a3c845a5ca22e6, State: Running, Role: LEADER
21:21:59.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:59.451367 10505 consensus_queue.cc:237] T 00000000000000000000000000000000 P e2d8a542014049bf87a3c845a5ca22e6 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cf87ba5a13b14d2dbd2901abf3152ee0" member_type: VOTER last_known_addr { host: "127.10.5.126" port: 42447 } } peers { permanent_uuid: "c789c3f650204bf4893740b2759914c9" member_type: VOTER last_known_addr { host: "127.10.5.125" port: 35615 } } peers { permanent_uuid: "e2d8a542014049bf87a3c845a5ca22e6" member_type: VOTER last_known_addr { host: "127.10.5.124" port: 43343 } }
21:21:59.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:59.459026 10530 sys_catalog.cc:455] T 00000000000000000000000000000000 P e2d8a542014049bf87a3c845a5ca22e6 [sys.catalog]: SysCatalogTable state changed. Reason: New leader e2d8a542014049bf87a3c845a5ca22e6. Latest consensus state: current_term: 1 leader_uuid: "e2d8a542014049bf87a3c845a5ca22e6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cf87ba5a13b14d2dbd2901abf3152ee0" member_type: VOTER last_known_addr { host: "127.10.5.126" port: 42447 } } peers { permanent_uuid: "c789c3f650204bf4893740b2759914c9" member_type: VOTER last_known_addr { host: "127.10.5.125" port: 35615 } } peers { permanent_uuid: "e2d8a542014049bf87a3c845a5ca22e6" member_type: VOTER last_known_addr { host: "127.10.5.124" port: 43343 } } }
21:21:59.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:59.459785 10530 sys_catalog.cc:458] T 00000000000000000000000000000000 P e2d8a542014049bf87a3c845a5ca22e6 [sys.catalog]: This master's current role is: LEADER
21:21:59.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:59.468253 10535 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
21:21:59.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:59.474509 10536 catalog_manager.cc:805] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
21:21:59.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:59.475057 10535 catalog_manager.cc:1494] Initializing Kudu cluster ID...
21:21:59.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:59.492738 10391 raft_consensus.cc:1275] T 00000000000000000000000000000000 P c789c3f650204bf4893740b2759914c9 [term 1 FOLLOWER]: Refusing update from remote peer e2d8a542014049bf87a3c845a5ca22e6: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
21:21:59.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:59.494923 10530 consensus_queue.cc:1048] T 00000000000000000000000000000000 P e2d8a542014049bf87a3c845a5ca22e6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c789c3f650204bf4893740b2759914c9" member_type: VOTER last_known_addr { host: "127.10.5.125" port: 35615 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
21:21:59.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:59.481069 10343 master_runner.cc:186] Error getting master registration for 127.10.5.126:42447: OK, instance_id { permanent_uuid: "cf87ba5a13b14d2dbd2901abf3152ee0" instance_seqno: 1769635314867051 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
21:21:59.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:59.512738 10343 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
21:21:59.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:59.520376 10522 sys_catalog.cc:455] T 00000000000000000000000000000000 P c789c3f650204bf4893740b2759914c9 [sys.catalog]: SysCatalogTable state changed. Reason: New leader e2d8a542014049bf87a3c845a5ca22e6. Latest consensus state: current_term: 1 leader_uuid: "e2d8a542014049bf87a3c845a5ca22e6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cf87ba5a13b14d2dbd2901abf3152ee0" member_type: VOTER last_known_addr { host: "127.10.5.126" port: 42447 } } peers { permanent_uuid: "c789c3f650204bf4893740b2759914c9" member_type: VOTER last_known_addr { host: "127.10.5.125" port: 35615 } } peers { permanent_uuid: "e2d8a542014049bf87a3c845a5ca22e6" member_type: VOTER last_known_addr { host: "127.10.5.124" port: 43343 } } }
21:21:59.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:59.521342 10522 sys_catalog.cc:458] T 00000000000000000000000000000000 P c789c3f650204bf4893740b2759914c9 [sys.catalog]: This master's current role is: FOLLOWER
21:21:59.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:59.527972 10537 mvcc.cc:204] Tried to move back new op lower bound from 7248426268625870848 to 7248426268482600960. Current Snapshot: MvccSnapshot[applied={T|T < 7248426268625870848}]
21:21:59.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:59.529881 10505 sys_catalog.cc:455] T 00000000000000000000000000000000 P e2d8a542014049bf87a3c845a5ca22e6 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "e2d8a542014049bf87a3c845a5ca22e6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cf87ba5a13b14d2dbd2901abf3152ee0" member_type: VOTER last_known_addr { host: "127.10.5.126" port: 42447 } } peers { permanent_uuid: "c789c3f650204bf4893740b2759914c9" member_type: VOTER last_known_addr { host: "127.10.5.125" port: 35615 } } peers { permanent_uuid: "e2d8a542014049bf87a3c845a5ca22e6" member_type: VOTER last_known_addr { host: "127.10.5.124" port: 43343 } } }
21:21:59.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:59.530603 10505 sys_catalog.cc:458] T 00000000000000000000000000000000 P e2d8a542014049bf87a3c845a5ca22e6 [sys.catalog]: This master's current role is: LEADER
21:21:59.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:59.533121 10522 sys_catalog.cc:455] T 00000000000000000000000000000000 P c789c3f650204bf4893740b2759914c9 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "e2d8a542014049bf87a3c845a5ca22e6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cf87ba5a13b14d2dbd2901abf3152ee0" member_type: VOTER last_known_addr { host: "127.10.5.126" port: 42447 } } peers { permanent_uuid: "c789c3f650204bf4893740b2759914c9" member_type: VOTER last_known_addr { host: "127.10.5.125" port: 35615 } } peers { permanent_uuid: "e2d8a542014049bf87a3c845a5ca22e6" member_type: VOTER last_known_addr { host: "127.10.5.124" port: 43343 } } }
21:21:59.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:59.533775 10522 sys_catalog.cc:458] T 00000000000000000000000000000000 P c789c3f650204bf4893740b2759914c9 [sys.catalog]: This master's current role is: FOLLOWER
21:21:59.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:59.536536 10535 catalog_manager.cc:1357] Generated new cluster ID: 9cdd81e1bdee4acf82c329a2d81f12f8
21:21:59.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:59.536815 10535 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
21:21:59.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:59.542171 10341 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.5.124" port: 43343 } attempt: 7
21:21:59.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:59.578518 10341 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P cf87ba5a13b14d2dbd2901abf3152ee0: Bootstrap starting.
21:21:59.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:59.585615 10341 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P cf87ba5a13b14d2dbd2901abf3152ee0: Neither blocks nor log segments found. Creating new log.
21:21:59.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:59.587859 10341 log.cc:826] T 00000000000000000000000000000000 P cf87ba5a13b14d2dbd2901abf3152ee0: Log is configured to *not* fsync() on all Append() calls
21:21:59.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:59.589205 10535 catalog_manager.cc:1380] Generated new certificate authority record
21:21:59.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:59.591276 10535 catalog_manager.cc:1514] Loading token signing keys...
21:21:59.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:59.611737 10341 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P cf87ba5a13b14d2dbd2901abf3152ee0: No bootstrap required, opened a new log
21:21:59.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:59.631444 10535 catalog_manager.cc:6027] T 00000000000000000000000000000000 P e2d8a542014049bf87a3c845a5ca22e6: Generated new TSK 0
21:21:59.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:59.632776 10535 catalog_manager.cc:1524] Initializing in-progress tserver states...
21:21:59.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:59.642979 10341 raft_consensus.cc:359] T 00000000000000000000000000000000 P cf87ba5a13b14d2dbd2901abf3152ee0 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cf87ba5a13b14d2dbd2901abf3152ee0" member_type: VOTER last_known_addr { host: "127.10.5.126" port: 42447 } } peers { permanent_uuid: "c789c3f650204bf4893740b2759914c9" member_type: VOTER last_known_addr { host: "127.10.5.125" port: 35615 } } peers { permanent_uuid: "e2d8a542014049bf87a3c845a5ca22e6" member_type: VOTER last_known_addr { host: "127.10.5.124" port: 43343 } }
21:21:59.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:59.643877 10341 raft_consensus.cc:385] T 00000000000000000000000000000000 P cf87ba5a13b14d2dbd2901abf3152ee0 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:21:59.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:59.644181 10341 raft_consensus.cc:740] T 00000000000000000000000000000000 P cf87ba5a13b14d2dbd2901abf3152ee0 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: cf87ba5a13b14d2dbd2901abf3152ee0, State: Initialized, Role: FOLLOWER
21:21:59.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:59.645061 10341 consensus_queue.cc:260] T 00000000000000000000000000000000 P cf87ba5a13b14d2dbd2901abf3152ee0 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cf87ba5a13b14d2dbd2901abf3152ee0" member_type: VOTER last_known_addr { host: "127.10.5.126" port: 42447 } } peers { permanent_uuid: "c789c3f650204bf4893740b2759914c9" member_type: VOTER last_known_addr { host: "127.10.5.125" port: 35615 } } peers { permanent_uuid: "e2d8a542014049bf87a3c845a5ca22e6" member_type: VOTER last_known_addr { host: "127.10.5.124" port: 43343 } }
21:21:59.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:59.647338 10544 sys_catalog.cc:455] T 00000000000000000000000000000000 P cf87ba5a13b14d2dbd2901abf3152ee0 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cf87ba5a13b14d2dbd2901abf3152ee0" member_type: VOTER last_known_addr { host: "127.10.5.126" port: 42447 } } peers { permanent_uuid: "c789c3f650204bf4893740b2759914c9" member_type: VOTER last_known_addr { host: "127.10.5.125" port: 35615 } } peers { permanent_uuid: "e2d8a542014049bf87a3c845a5ca22e6" member_type: VOTER last_known_addr { host: "127.10.5.124" port: 43343 } } }
21:21:59.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:59.648126 10544 sys_catalog.cc:458] T 00000000000000000000000000000000 P cf87ba5a13b14d2dbd2901abf3152ee0 [sys.catalog]: This master's current role is: FOLLOWER
21:21:59.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:59.649377 10341 sys_catalog.cc:565] T 00000000000000000000000000000000 P cf87ba5a13b14d2dbd2901abf3152ee0 [sys.catalog]: configured and running, proceeding with master startup.
21:21:59.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:59.649556 10315 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "c789c3f650204bf4893740b2759914c9" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "cf87ba5a13b14d2dbd2901abf3152ee0" is_pre_election: true
21:21:59.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:59.650327 10315 raft_consensus.cc:2468] T 00000000000000000000000000000000 P cf87ba5a13b14d2dbd2901abf3152ee0 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c789c3f650204bf4893740b2759914c9 in term 0.
21:21:59.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:59.651414 10314 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "e2d8a542014049bf87a3c845a5ca22e6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "cf87ba5a13b14d2dbd2901abf3152ee0"
21:21:59.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:59.651952 10366 leader_election.cc:304] T 00000000000000000000000000000000 P c789c3f650204bf4893740b2759914c9 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: c789c3f650204bf4893740b2759914c9, cf87ba5a13b14d2dbd2901abf3152ee0; no voters: e2d8a542014049bf87a3c845a5ca22e6
21:21:59.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:59.653470 10522 raft_consensus.cc:2764] T 00000000000000000000000000000000 P c789c3f650204bf4893740b2759914c9 [term 1 FOLLOWER]: Leader pre-election decision vote started in defunct term 0: won
21:21:59.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:59.652985 10316 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "e2d8a542014049bf87a3c845a5ca22e6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "cf87ba5a13b14d2dbd2901abf3152ee0" is_pre_election: true
21:21:59.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:59.652493 10313 raft_consensus.cc:3060] T 00000000000000000000000000000000 P cf87ba5a13b14d2dbd2901abf3152ee0 [term 0 FOLLOWER]: Advancing to term 1
21:21:59.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:59.658726 10261 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskp7EThw/build/tsan/bin/kudu
21:21:59.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskp7EThw/build/tsan/bin/kudu
21:21:59.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster5151574767910532771/ts-0/wal
21:21:59.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster5151574767910532771/ts-0/data
21:21:59.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:21:59.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:21:59.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:21:59.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:21:59.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:21:59.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:21:59.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster5151574767910532771/ts-0/logs
21:21:59.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster5151574767910532771/ts-0/data/info.pb
21:21:59.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:21:59.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:21:59.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:21:59.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:21:59.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:21:59.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:21:59.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:21:59.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:21:59.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
21:21:59.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:21:59.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.5.65:0
21:21:59.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.10.5.65
21:21:59.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.5.65
21:21:59.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:21:59.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.10.5.126:42447,127.10.5.125:35615,127.10.5.124:43343
21:21:59.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.5.84:40825
21:21:59.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:21:59.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:21:59.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
21:21:59.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:59.665625 10313 raft_consensus.cc:1275] T 00000000000000000000000000000000 P cf87ba5a13b14d2dbd2901abf3152ee0 [term 1 FOLLOWER]: Refusing update from remote peer e2d8a542014049bf87a3c845a5ca22e6: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
21:21:59.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:59.667261 10530 consensus_queue.cc:1048] T 00000000000000000000000000000000 P e2d8a542014049bf87a3c845a5ca22e6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "cf87ba5a13b14d2dbd2901abf3152ee0" member_type: VOTER last_known_addr { host: "127.10.5.126" port: 42447 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
21:21:59.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:59.785986 10551 sys_catalog.cc:455] T 00000000000000000000000000000000 P cf87ba5a13b14d2dbd2901abf3152ee0 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "e2d8a542014049bf87a3c845a5ca22e6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cf87ba5a13b14d2dbd2901abf3152ee0" member_type: VOTER last_known_addr { host: "127.10.5.126" port: 42447 } } peers { permanent_uuid: "c789c3f650204bf4893740b2759914c9" member_type: VOTER last_known_addr { host: "127.10.5.125" port: 35615 } } peers { permanent_uuid: "e2d8a542014049bf87a3c845a5ca22e6" member_type: VOTER last_known_addr { host: "127.10.5.124" port: 43343 } } }
21:21:59.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:59.787660 10544 sys_catalog.cc:455] T 00000000000000000000000000000000 P cf87ba5a13b14d2dbd2901abf3152ee0 [sys.catalog]: SysCatalogTable state changed. Reason: New leader e2d8a542014049bf87a3c845a5ca22e6. Latest consensus state: current_term: 1 leader_uuid: "e2d8a542014049bf87a3c845a5ca22e6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cf87ba5a13b14d2dbd2901abf3152ee0" member_type: VOTER last_known_addr { host: "127.10.5.126" port: 42447 } } peers { permanent_uuid: "c789c3f650204bf4893740b2759914c9" member_type: VOTER last_known_addr { host: "127.10.5.125" port: 35615 } } peers { permanent_uuid: "e2d8a542014049bf87a3c845a5ca22e6" member_type: VOTER last_known_addr { host: "127.10.5.124" port: 43343 } } }
21:21:59.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:59.788489 10544 sys_catalog.cc:458] T 00000000000000000000000000000000 P cf87ba5a13b14d2dbd2901abf3152ee0 [sys.catalog]: This master's current role is: FOLLOWER
21:21:59.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:59.789217 10551 sys_catalog.cc:458] T 00000000000000000000000000000000 P cf87ba5a13b14d2dbd2901abf3152ee0 [sys.catalog]: This master's current role is: FOLLOWER
21:21:59.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:59.835937 10561 catalog_manager.cc:1269] Loaded cluster ID: 9cdd81e1bdee4acf82c329a2d81f12f8
21:21:59.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:59.839807 10561 catalog_manager.cc:1562] T 00000000000000000000000000000000 P cf87ba5a13b14d2dbd2901abf3152ee0: loading cluster ID for follower catalog manager: success
21:21:59.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:59.846999 10561 catalog_manager.cc:1584] T 00000000000000000000000000000000 P cf87ba5a13b14d2dbd2901abf3152ee0: acquiring CA information for follower catalog manager: success
21:21:59.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:21:59.852351 10561 catalog_manager.cc:1612] T 00000000000000000000000000000000 P cf87ba5a13b14d2dbd2901abf3152ee0: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
21:22:00.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:00.055440 10530 sys_catalog.cc:455] T 00000000000000000000000000000000 P e2d8a542014049bf87a3c845a5ca22e6 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "e2d8a542014049bf87a3c845a5ca22e6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cf87ba5a13b14d2dbd2901abf3152ee0" member_type: VOTER last_known_addr { host: "127.10.5.126" port: 42447 } } peers { permanent_uuid: "c789c3f650204bf4893740b2759914c9" member_type: VOTER last_known_addr { host: "127.10.5.125" port: 35615 } } peers { permanent_uuid: "e2d8a542014049bf87a3c845a5ca22e6" member_type: VOTER last_known_addr { host: "127.10.5.124" port: 43343 } } }
21:22:00.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:00.056162 10530 sys_catalog.cc:458] T 00000000000000000000000000000000 P e2d8a542014049bf87a3c845a5ca22e6 [sys.catalog]: This master's current role is: LEADER
21:22:00.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:00.134807 10546 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:22:00.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:00.135445 10546 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:22:00.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:00.136221 10546 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:22:00.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:00.194357 10546 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:22:00.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:00.195827 10546 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.10.5.65
21:22:00.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:00.259112 10546 tablet_server_runner.cc:78] Tablet server non-default flags:
21:22:00.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:22:00.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.5.84:40825
21:22:00.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:22:00.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:22:00.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster5151574767910532771/ts-0/data
21:22:00.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster5151574767910532771/ts-0/wal
21:22:00.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:22:00.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:22:00.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.5.65:0
21:22:00.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:22:00.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:22:00.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:22:00.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster5151574767910532771/ts-0/data/info.pb
21:22:00.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.5.65
21:22:00.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:22:00.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.10.5.126:42447,127.10.5.125:35615,127.10.5.124:43343
21:22:00.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:22:00.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:22:00.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:22:00.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:22:00.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:22:00.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.10.5.65
21:22:00.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster5151574767910532771/ts-0/logs
21:22:00.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:22:00.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:22:00.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:22:00.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
21:22:00.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:22:00.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3722ed49c19ce54c7feae2fa944e32b1d4ed8d2d
21:22:00.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:22:00.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 28 Jan 2026 21:02:00 UTC on 24a791456cd2
21:22:00.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10237
21:22:00.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:22:00.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:00.260857 10546 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:22:00.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:00.262952 10546 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:22:00.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:00.278970 10568 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:22:00.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:00.499037 10536 catalog_manager.cc:1269] Loaded cluster ID: 9cdd81e1bdee4acf82c329a2d81f12f8
21:22:00.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:00.502053 10536 catalog_manager.cc:1562] T 00000000000000000000000000000000 P c789c3f650204bf4893740b2759914c9: loading cluster ID for follower catalog manager: success
21:22:00.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:00.518576 10536 catalog_manager.cc:1584] T 00000000000000000000000000000000 P c789c3f650204bf4893740b2759914c9: acquiring CA information for follower catalog manager: success
21:22:00.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:00.555979 10536 catalog_manager.cc:1612] T 00000000000000000000000000000000 P c789c3f650204bf4893740b2759914c9: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
21:22:02.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:01.680490 10567 debug-util.cc:398] Leaking SignalData structure 0x7b080002ec60 after lost signal to thread 10546
21:22:02.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:02.345391 10546 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 2.065s user 0.737s sys 1.213s
21:22:02.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:02.345764 10546 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 2.065s user 0.737s sys 1.213s
21:22:02.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:00.280925 10569 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:22:02.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:02.347678 10571 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:22:02.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:02.348359 10570 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2064 milliseconds
21:22:02.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:02.349099 10546 server_base.cc:1055] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:22:02.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:02.353303 10546 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:22:02.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:02.356384 10546 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:22:02.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:02.357956 10546 hybrid_clock.cc:648] HybridClock initialized: now 1769635322357910 us; error 51 us; skew 500 ppm
21:22:02.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:02.359155 10546 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:22:02.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:02.367049 10546 webserver.cc:492] Webserver started at http://127.10.5.65:46165/ using document root <none> and password file <none>
21:22:02.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:02.368662 10546 fs_manager.cc:362] Metadata directory not provided
21:22:02.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:02.369078 10546 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:22:02.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:02.369740 10546 server_base.cc:908] This appears to be a new deployment of Kudu; creating new FS layout
21:22:02.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:02.377112 10546 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster5151574767910532771/ts-0/data/instance:
21:22:02.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6d26332ba6dd47edb67689b7d4300d88"
21:22:02.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-01-28 21:22:02 on dist-test-slave-hq3s"
21:22:02.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:02.379009 10546 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster5151574767910532771/ts-0/wal/instance:
21:22:02.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6d26332ba6dd47edb67689b7d4300d88"
21:22:02.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-01-28 21:22:02 on dist-test-slave-hq3s"
21:22:02.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:02.389142 10546 fs_manager.cc:696] Time spent creating directory manager: real 0.009s user 0.007s sys 0.001s
21:22:02.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:02.398422 10579 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:22:02.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:02.399906 10546 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.002s sys 0.001s
21:22:02.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:02.400301 10546 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster5151574767910532771/ts-0/data,/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster5151574767910532771/ts-0/wal
21:22:02.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6d26332ba6dd47edb67689b7d4300d88"
21:22:02.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-01-28 21:22:02 on dist-test-slave-hq3s"
21:22:02.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:02.400741 10546 fs_report.cc:389] FS layout report
21:22:02.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:22:02.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster5151574767910532771/ts-0/wal
21:22:02.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster5151574767910532771/ts-0/wal
21:22:02.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster5151574767910532771/ts-0/data/data
21:22:02.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:22:02.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:22:02.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:22:02.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:22:02.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:22:02.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:22:02.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:22:02.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:22:02.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:22:02.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:22:02.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:22:02.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:02.469364 10546 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:22:02.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:02.471419 10546 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:22:02.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:02.472030 10546 kserver.cc:163] Server-wide thread pool size limit: 3276
21:22:02.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:02.475497 10546 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
21:22:02.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:02.481556 10546 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
21:22:02.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:02.481865 10546 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
21:22:02.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:02.482169 10546 ts_tablet_manager.cc:616] Registered 0 tablets
21:22:02.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:02.482384 10546 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
21:22:02.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:02.660571 10546 rpc_server.cc:307] RPC server started. Bound to: 127.10.5.65:35653
21:22:02.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:02.660712 10691 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.10.5.65:35653 every 8 connection(s)
21:22:02.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:02.664152 10546 server_base.cc:1192] Dumped server information to /tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster5151574767910532771/ts-0/data/info.pb
21:22:02.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:02.673882 10261 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskp7EThw/build/tsan/bin/kudu as pid 10546
21:22:02.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:02.674342 10261 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster5151574767910532771/ts-0/wal/instance
21:22:02.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:02.696144 10261 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskp7EThw/build/tsan/bin/kudu
21:22:02.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskp7EThw/build/tsan/bin/kudu
21:22:02.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster5151574767910532771/ts-1/wal
21:22:02.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster5151574767910532771/ts-1/data
21:22:02.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:22:02.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:22:02.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:22:02.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:22:02.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:22:02.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:22:02.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster5151574767910532771/ts-1/logs
21:22:02.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster5151574767910532771/ts-1/data/info.pb
21:22:02.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:22:02.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:22:02.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:22:02.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:22:02.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:22:02.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:22:02.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:22:02.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:22:02.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
21:22:02.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:22:02.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.5.66:0
21:22:02.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.10.5.66
21:22:02.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.5.66
21:22:02.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:22:02.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.10.5.126:42447,127.10.5.125:35615,127.10.5.124:43343
21:22:02.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.5.84:40825
21:22:02.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:22:02.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:22:02.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
21:22:02.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:02.801594 10693 heartbeater.cc:344] Connected to a master server at 127.10.5.125:35615
21:22:02.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:02.802065 10693 heartbeater.cc:461] Registering TS with master...
21:22:02.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:02.803325 10693 heartbeater.cc:507] Master 127.10.5.125:35615 requested a full tablet report, sending...
21:22:02.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:02.807580 10692 heartbeater.cc:344] Connected to a master server at 127.10.5.124:43343
21:22:02.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:02.808060 10692 heartbeater.cc:461] Registering TS with master...
21:22:02.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:02.808945 10692 heartbeater.cc:507] Master 127.10.5.124:43343 requested a full tablet report, sending...
21:22:02.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:02.812994 10461 ts_manager.cc:194] Registered new tserver with Master: 6d26332ba6dd47edb67689b7d4300d88 (127.10.5.65:35653)
21:22:02.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:02.816037 10461 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.10.5.65:36281
21:22:02.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:02.819303 10381 ts_manager.cc:194] Registered new tserver with Master: 6d26332ba6dd47edb67689b7d4300d88 (127.10.5.65:35653)
21:22:02.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:02.827572 10698 heartbeater.cc:344] Connected to a master server at 127.10.5.126:42447
21:22:02.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:02.828032 10698 heartbeater.cc:461] Registering TS with master...
21:22:02.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:02.828761 10698 heartbeater.cc:507] Master 127.10.5.126:42447 requested a full tablet report, sending...
21:22:02.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:02.831832 10305 ts_manager.cc:194] Registered new tserver with Master: 6d26332ba6dd47edb67689b7d4300d88 (127.10.5.65:35653)
21:22:03.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:03.224007 10697 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:22:03.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:03.224578 10697 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:22:03.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:03.225148 10697 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:22:03.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:03.257983 10697 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:22:03.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:03.258883 10697 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.10.5.66
21:22:03.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:03.295195 10697 tablet_server_runner.cc:78] Tablet server non-default flags:
21:22:03.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:22:03.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.5.84:40825
21:22:03.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:22:03.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:22:03.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster5151574767910532771/ts-1/data
21:22:03.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster5151574767910532771/ts-1/wal
21:22:03.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:22:03.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:22:03.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.5.66:0
21:22:03.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:22:03.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:22:03.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:22:03.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster5151574767910532771/ts-1/data/info.pb
21:22:03.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.5.66
21:22:03.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:22:03.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.10.5.126:42447,127.10.5.125:35615,127.10.5.124:43343
21:22:03.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:22:03.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:22:03.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:22:03.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:22:03.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:22:03.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.10.5.66
21:22:03.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster5151574767910532771/ts-1/logs
21:22:03.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:22:03.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:22:03.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:22:03.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
21:22:03.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:22:03.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3722ed49c19ce54c7feae2fa944e32b1d4ed8d2d
21:22:03.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:22:03.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 28 Jan 2026 21:02:00 UTC on 24a791456cd2
21:22:03.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10237
21:22:03.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:22:03.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:03.296784 10697 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:22:03.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:03.298417 10697 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:22:03.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:03.312599 10710 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:22:03.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:03.825477 10692 heartbeater.cc:499] Master 127.10.5.124:43343 was elected leader, sending a full tablet report...
21:22:04.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:04.713866 10709 debug-util.cc:398] Leaking SignalData structure 0x7b080002ec60 after lost signal to thread 10697
21:22:04.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:04.901698 10697 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.588s user 0.640s sys 0.859s
21:22:04.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:04.902073 10697 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.589s user 0.640s sys 0.859s
21:22:04.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:03.314800 10711 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:22:04.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:04.903993 10713 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:22:04.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:04.907192 10712 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1590 milliseconds
21:22:04.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:04.907254 10697 server_base.cc:1055] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:22:04.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:04.908617 10697 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:22:04.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:04.910715 10697 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:22:04.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:04.912060 10697 hybrid_clock.cc:648] HybridClock initialized: now 1769635324912020 us; error 49 us; skew 500 ppm
21:22:04.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:04.913015 10697 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:22:04.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:04.918982 10697 webserver.cc:492] Webserver started at http://127.10.5.66:36829/ using document root <none> and password file <none>
21:22:04.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:04.920192 10697 fs_manager.cc:362] Metadata directory not provided
21:22:04.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:04.920408 10697 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:22:04.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:04.920874 10697 server_base.cc:908] This appears to be a new deployment of Kudu; creating new FS layout
21:22:04.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:04.924995 10697 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster5151574767910532771/ts-1/data/instance:
21:22:04.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "52c97435edee4fada03a70647d15542b"
21:22:04.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-01-28 21:22:04 on dist-test-slave-hq3s"
21:22:04.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:04.926080 10697 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster5151574767910532771/ts-1/wal/instance:
21:22:04.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "52c97435edee4fada03a70647d15542b"
21:22:04.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-01-28 21:22:04 on dist-test-slave-hq3s"
21:22:04.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:04.933372 10697 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.004s sys 0.004s
21:22:04.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:04.938903 10720 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:22:04.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:04.939970 10697 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.002s sys 0.001s
21:22:04.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:04.940285 10697 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster5151574767910532771/ts-1/data,/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster5151574767910532771/ts-1/wal
21:22:04.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "52c97435edee4fada03a70647d15542b"
21:22:04.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-01-28 21:22:04 on dist-test-slave-hq3s"
21:22:04.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:04.940781 10697 fs_report.cc:389] FS layout report
21:22:04.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:22:04.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster5151574767910532771/ts-1/wal
21:22:04.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster5151574767910532771/ts-1/wal
21:22:04.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster5151574767910532771/ts-1/data/data
21:22:04.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:22:04.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:22:04.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:22:04.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:22:04.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:22:04.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:22:04.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:22:04.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:22:04.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:22:04.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:22:04.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:22:04.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:04.991681 10697 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:22:04.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:04.993587 10697 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:22:04.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:04.994112 10697 kserver.cc:163] Server-wide thread pool size limit: 3276
21:22:04.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:04.997334 10697 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
21:22:05.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:05.002966 10697 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
21:22:05.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:05.003276 10697 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
21:22:05.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:05.003633 10697 ts_tablet_manager.cc:616] Registered 0 tablets
21:22:05.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:05.003866 10697 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.001s sys 0.000s
21:22:05.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:05.133451 10697 rpc_server.cc:307] RPC server started. Bound to: 127.10.5.66:32781
21:22:05.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:05.133551 10832 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.10.5.66:32781 every 8 connection(s)
21:22:05.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:05.136186 10697 server_base.cc:1192] Dumped server information to /tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster5151574767910532771/ts-1/data/info.pb
21:22:05.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:05.142998 10261 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskp7EThw/build/tsan/bin/kudu as pid 10697
21:22:05.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:05.143710 10261 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster5151574767910532771/ts-1/wal/instance
21:22:05.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:05.158752 10261 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskp7EThw/build/tsan/bin/kudu
21:22:05.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskp7EThw/build/tsan/bin/kudu
21:22:05.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster5151574767910532771/ts-2/wal
21:22:05.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster5151574767910532771/ts-2/data
21:22:05.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:22:05.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:22:05.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:22:05.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:22:05.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:22:05.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:22:05.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster5151574767910532771/ts-2/logs
21:22:05.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster5151574767910532771/ts-2/data/info.pb
21:22:05.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:22:05.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:22:05.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:22:05.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:22:05.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:22:05.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:22:05.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:22:05.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:22:05.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
21:22:05.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:22:05.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.5.67:0
21:22:05.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.10.5.67
21:22:05.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.5.67
21:22:05.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:22:05.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.10.5.126:42447,127.10.5.125:35615,127.10.5.124:43343
21:22:05.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.5.84:40825
21:22:05.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:22:05.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:22:05.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
21:22:05.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:05.189621 10833 heartbeater.cc:344] Connected to a master server at 127.10.5.124:43343
21:22:05.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:05.190162 10833 heartbeater.cc:461] Registering TS with master...
21:22:05.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:05.190271 10835 heartbeater.cc:344] Connected to a master server at 127.10.5.126:42447
21:22:05.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:05.190662 10835 heartbeater.cc:461] Registering TS with master...
21:22:05.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:05.191632 10833 heartbeater.cc:507] Master 127.10.5.124:43343 requested a full tablet report, sending...
21:22:05.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:05.191862 10835 heartbeater.cc:507] Master 127.10.5.126:42447 requested a full tablet report, sending...
21:22:05.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:05.194412 10461 ts_manager.cc:194] Registered new tserver with Master: 52c97435edee4fada03a70647d15542b (127.10.5.66:32781)
21:22:05.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:05.194991 10305 ts_manager.cc:194] Registered new tserver with Master: 52c97435edee4fada03a70647d15542b (127.10.5.66:32781)
21:22:05.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:05.196173 10461 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.10.5.66:48755
21:22:05.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:05.206418 10834 heartbeater.cc:344] Connected to a master server at 127.10.5.125:35615
21:22:05.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:05.206830 10834 heartbeater.cc:461] Registering TS with master...
21:22:05.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:05.207397 10834 heartbeater.cc:507] Master 127.10.5.125:35615 requested a full tablet report, sending...
21:22:05.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:05.210039 10381 ts_manager.cc:194] Registered new tserver with Master: 52c97435edee4fada03a70647d15542b (127.10.5.66:32781)
21:22:05.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:05.475488 10841 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:22:05.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:05.476059 10841 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:22:05.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:05.476729 10841 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:22:05.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:05.509099 10841 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:22:05.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:05.509997 10841 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.10.5.67
21:22:05.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:05.545676 10841 tablet_server_runner.cc:78] Tablet server non-default flags:
21:22:05.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:22:05.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.5.84:40825
21:22:05.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:22:05.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:22:05.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster5151574767910532771/ts-2/data
21:22:05.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster5151574767910532771/ts-2/wal
21:22:05.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:22:05.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:22:05.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.5.67:0
21:22:05.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:22:05.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:22:05.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:22:05.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster5151574767910532771/ts-2/data/info.pb
21:22:05.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.5.67
21:22:05.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:22:05.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.10.5.126:42447,127.10.5.125:35615,127.10.5.124:43343
21:22:05.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:22:05.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:22:05.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:22:05.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:22:05.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:22:05.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.10.5.67
21:22:05.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster5151574767910532771/ts-2/logs
21:22:05.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:22:05.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:22:05.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:22:05.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
21:22:05.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:22:05.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3722ed49c19ce54c7feae2fa944e32b1d4ed8d2d
21:22:05.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:22:05.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 28 Jan 2026 21:02:00 UTC on 24a791456cd2
21:22:05.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10237
21:22:05.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:22:05.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:05.547212 10841 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:22:05.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:05.548825 10841 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:22:05.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:05.560832 10850 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:22:06.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:06.200054 10833 heartbeater.cc:499] Master 127.10.5.124:43343 was elected leader, sending a full tablet report...
21:22:06.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:05.564806 10853 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:22:06.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:05.562399 10851 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:22:06.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:06.705456 10852 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1143 milliseconds
21:22:06.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:06.705578 10841 server_base.cc:1055] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:22:06.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:06.707060 10841 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:22:06.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:06.709662 10841 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:22:06.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:06.711259 10841 hybrid_clock.cc:648] HybridClock initialized: now 1769635326711203 us; error 47 us; skew 500 ppm
21:22:06.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:06.712378 10841 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:22:06.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:06.718968 10841 webserver.cc:492] Webserver started at http://127.10.5.67:39177/ using document root <none> and password file <none>
21:22:06.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:06.720018 10841 fs_manager.cc:362] Metadata directory not provided
21:22:06.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:06.720234 10841 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:22:06.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:06.720631 10841 server_base.cc:908] This appears to be a new deployment of Kudu; creating new FS layout
21:22:06.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:06.724831 10841 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster5151574767910532771/ts-2/data/instance:
21:22:06.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3b95157c15794b55b187338a1fd08c9b"
21:22:06.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-01-28 21:22:06 on dist-test-slave-hq3s"
21:22:06.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:06.725919 10841 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster5151574767910532771/ts-2/wal/instance:
21:22:06.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3b95157c15794b55b187338a1fd08c9b"
21:22:06.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-01-28 21:22:06 on dist-test-slave-hq3s"
21:22:06.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:06.733088 10841 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.004s sys 0.004s
21:22:06.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:06.739163 10860 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:22:06.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:06.740453 10841 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.001s
21:22:06.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:06.740767 10841 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster5151574767910532771/ts-2/data,/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster5151574767910532771/ts-2/wal
21:22:06.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3b95157c15794b55b187338a1fd08c9b"
21:22:06.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-01-28 21:22:06 on dist-test-slave-hq3s"
21:22:06.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:06.741078 10841 fs_report.cc:389] FS layout report
21:22:06.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:22:06.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster5151574767910532771/ts-2/wal
21:22:06.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster5151574767910532771/ts-2/wal
21:22:06.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster5151574767910532771/ts-2/data/data
21:22:06.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:22:06.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:22:06.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:22:06.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:22:06.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:22:06.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:22:06.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:22:06.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:22:06.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:22:06.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:22:06.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:22:06.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:06.790091 10841 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:22:06.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:06.791512 10841 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:22:06.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:06.791958 10841 kserver.cc:163] Server-wide thread pool size limit: 3276
21:22:06.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:06.794461 10841 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
21:22:06.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:06.798592 10841 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
21:22:06.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:06.798806 10841 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
21:22:06.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:06.799017 10841 ts_tablet_manager.cc:616] Registered 0 tablets
21:22:06.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:06.799150 10841 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
21:22:06.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:06.934209 10841 rpc_server.cc:307] RPC server started. Bound to: 127.10.5.67:41847
21:22:06.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:06.934305 10972 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.10.5.67:41847 every 8 connection(s)
21:22:06.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:06.936874 10841 server_base.cc:1192] Dumped server information to /tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster5151574767910532771/ts-2/data/info.pb
21:22:06.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:06.939896 10261 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskp7EThw/build/tsan/bin/kudu as pid 10841
21:22:06.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:06.940351 10261 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster5151574767910532771/ts-2/wal/instance
21:22:06.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:06.989414 10973 heartbeater.cc:344] Connected to a master server at 127.10.5.124:43343
21:22:06.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:06.989971 10973 heartbeater.cc:461] Registering TS with master...
21:22:06.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:06.991312 10973 heartbeater.cc:507] Master 127.10.5.124:43343 requested a full tablet report, sending...
21:22:06.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:06.993014 10974 heartbeater.cc:344] Connected to a master server at 127.10.5.125:35615
21:22:06.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:06.993330 10974 heartbeater.cc:461] Registering TS with master...
21:22:06.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:06.993963 10461 ts_manager.cc:194] Registered new tserver with Master: 3b95157c15794b55b187338a1fd08c9b (127.10.5.67:41847)
21:22:06.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:06.994210 10974 heartbeater.cc:507] Master 127.10.5.125:35615 requested a full tablet report, sending...
21:22:06.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:06.995647 10461 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.10.5.67:50619
21:22:06.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:06.997287 10381 ts_manager.cc:194] Registered new tserver with Master: 3b95157c15794b55b187338a1fd08c9b (127.10.5.67:41847)
21:22:07.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:06.999974 10975 heartbeater.cc:344] Connected to a master server at 127.10.5.126:42447
21:22:07.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:07.000236 10975 heartbeater.cc:461] Registering TS with master...
21:22:07.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:07.000769 10975 heartbeater.cc:507] Master 127.10.5.126:42447 requested a full tablet report, sending...
21:22:07.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:07.003075 10305 ts_manager.cc:194] Registered new tserver with Master: 3b95157c15794b55b187338a1fd08c9b (127.10.5.67:41847)
21:22:07.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:07.007479 10261 external_mini_cluster.cc:949] 3 TS(s) registered with all masters
21:22:07.017 [DEBUG - main] (MiniKuduCluster.java:195) Response:
21:22:07.017 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
21:22:07.018 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.10.5.126"
port: 42447
}
bound_http_address {
host: "127.10.5.126"
port: 43107
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.10.5.125"
port: 35615
}
bound_http_address {
host: "127.10.5.125"
port: 41027
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.10.5.124"
port: 43343
}
bound_http_address {
host: "127.10.5.124"
port: 41527
}
}
}
21:22:07.019 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
21:22:07.020 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.10.5.65"
port: 35653
}
bound_http_address {
host: "127.10.5.65"
port: 46165
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.10.5.66"
port: 32781
}
bound_http_address {
host: "127.10.5.66"
port: 36829
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.10.5.67"
port: 41847
}
bound_http_address {
host: "127.10.5.67"
port: 39177
}
}
}
21:22:07.020 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
21:22:07.021 [DEBUG - main] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster8803669063231258229/krb5kdc/krb5cc
21:22:07.022 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
21:22:07.022 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskp7EThw/test-tmp
21:22:07.023 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
21:22:07.023 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskp7EThw/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
21:22:07.024 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637"
extra_master_flags: "--webserver_require_spnego=false"
extra_master_flags: "--authn_token_validity_seconds=10"
extra_tserver_flags: "--webserver_require_spnego=false"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
21:22:07.345 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:22:07.345 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
21:22:07.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:07.348979 10984 mini_oidc.cc:150] Starting JWKS server
21:22:07.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:07.352169 10984 webserver.cc:492] Webserver started at http://127.0.0.1:34709/ using document root <none> and password file <none>
21:22:07.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:07.352615 10984 mini_oidc.cc:186] Starting OIDC Discovery server
21:22:07.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:07.354334 10984 webserver.cc:492] Webserver started at http://127.0.0.1:45063/ using document root <none> and password file <none>
21:22:07.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
21:22:07.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/krb5kdc/principal' for realm 'KRBTEST.COM',
21:22:07.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
21:22:07.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 28 21:22:07 dist-test-slave-hq3s krb5kdc[10996](info): setting up network...
21:22:07.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
21:22:07.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 28 21:22:07 dist-test-slave-hq3s krb5kdc[10996](info): set up 2 sockets
21:22:07.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 28 21:22:07 dist-test-slave-hq3s krb5kdc[10996](info): commencing operation
21:22:07.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
21:22:08.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:07.999521 10973 heartbeater.cc:499] Master 127.10.5.124:43343 was elected leader, sending a full tablet report...
21:22:09.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:09.283977 10984 mini_kdc.cc:121] Time spent starting KDC: real 1.896s user 0.003s sys 0.015s
21:22:09.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
21:22:09.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
21:22:09.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
21:22:09.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
21:22:09.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
21:22:09.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
21:22:09.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
21:22:09.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
21:22:09.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
21:22:09.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
21:22:09.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/krb5kdc/test-user.keytab.
21:22:09.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/krb5kdc/test-user.keytab.
21:22:09.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 28 21:22:09 dist-test-slave-hq3s krb5kdc[10996](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1769635329, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:22:09.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
21:22:09.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-01-28T21:22:09Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
21:22:09.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-01-28T21:22:09Z Disabled control of system clock
21:22:09.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.10.186.62@KRBTEST.COM; defaulting to no policy
21:22:09.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:22:09.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.10.186.62@KRBTEST.COM" created.
21:22:09.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:22:09.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.10.186.62 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/krb5kdc/kudu.keytab.
21:22:09.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.10.186.62 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/krb5kdc/kudu.keytab.
21:22:09.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.10.186.62@KRBTEST.COM; defaulting to no policy
21:22:09.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:22:09.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.10.186.62@KRBTEST.COM" created.
21:22:09.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:22:09.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.10.186.62 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/krb5kdc/kudu.keytab.
21:22:09.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.10.186.62 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/krb5kdc/kudu.keytab.
21:22:09.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:09.490348 10984 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskp7EThw/build/tsan/bin/kudu
21:22:09.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskp7EThw/build/tsan/bin/kudu
21:22:09.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/master-0/wal
21:22:09.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/master-0/data
21:22:09.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:22:09.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:22:09.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:22:09.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:22:09.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:22:09.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:22:09.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/master-0/logs
21:22:09.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/master-0/data/info.pb
21:22:09.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:22:09.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:22:09.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:22:09.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:22:09.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:22:09.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:22:09.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:22:09.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:22:09.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:22:09.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:22:09.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:22:09.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:22:09.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.186.62:46125
21:22:09.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.186.62
21:22:09.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:22:09.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/krb5kdc/kudu.keytab
21:22:09.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.10.186.62
21:22:09.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:22:09.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
21:22:09.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
21:22:09.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
21:22:09.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.186.20:34545
21:22:09.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:22:09.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:22:09.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:22:09.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:22:09.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.186.62:46125,127.10.186.61:45819,127.10.186.60:38107
21:22:09.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:22:09.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34709/jwks.json
21:22:09.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
21:22:09.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
21:22:09.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:09.783959 11012 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:22:09.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:09.784530 11012 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:22:09.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:09.785244 11012 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:22:09.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:09.818320 11012 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
21:22:09.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:09.818563 11012 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:22:09.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:09.818822 11012 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:22:09.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:09.819023 11012 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:22:09.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:09.819269 11012 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:22:09.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:09.819471 11012 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:22:09.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:09.819655 11012 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34709/jwks.json
21:22:09.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:09.859215 11012 master_runner.cc:386] Master server non-default flags:
21:22:09.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:22:09.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.186.20:34545
21:22:09.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:22:09.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:22:09.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/master-0/data
21:22:09.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/master-0/wal
21:22:09.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
21:22:09.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:22:09.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.186.62:46125,127.10.186.61:45819,127.10.186.60:38107
21:22:09.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/krb5kdc/kudu.keytab
21:22:09.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.10.186.62
21:22:09.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:22:09.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:22:09.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:22:09.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.186.62:46125
21:22:09.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:22:09.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:22:09.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:22:09.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34709/jwks.json
21:22:09.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:22:09.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
21:22:09.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
21:22:09.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:22:09.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:22:09.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/master-0/data/info.pb
21:22:09.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.186.62
21:22:09.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:22:09.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:22:09.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:22:09.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:22:09.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:22:09.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:22:09.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/master-0/logs
21:22:09.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:22:09.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:22:09.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:22:09.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:22:09.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:22:09.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3722ed49c19ce54c7feae2fa944e32b1d4ed8d2d
21:22:09.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:22:09.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 28 Jan 2026 21:02:00 UTC on 24a791456cd2
21:22:09.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10237
21:22:09.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:22:09.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:09.860492 11012 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:22:09.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:09.861469 11012 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:22:09.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:09.872905 11018 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:22:11.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:09.873363 11019 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:22:11.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:11.169406 11020 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1291 milliseconds
21:22:11.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:11.170470 11012 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.298s user 0.477s sys 0.800s
21:22:11.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:11.170714 11021 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:22:11.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:11.170863 11012 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.298s user 0.477s sys 0.800s
21:22:11.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:11.171168 11012 server_base.cc:1055] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:22:11.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:11.172714 11012 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:22:11.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:11.175748 11012 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:22:11.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:11.177187 11012 hybrid_clock.cc:648] HybridClock initialized: now 1769635331177138 us; error 40 us; skew 500 ppm
21:22:11.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:11.178288 11012 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:22:11.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 28 21:22:11 dist-test-slave-hq3s krb5kdc[10996](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1769635331, etypes {rep=17 tkt=17 ses=17}, kudu/127.10.186.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:22:11.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:11.186578 11012 init.cc:377] Logged in from keytab as kudu/127.10.186.62@KRBTEST.COM (short username kudu)
21:22:11.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:11.192546 11012 webserver.cc:492] Webserver started at http://127.10.186.62:37867/ using document root <none> and password file <none>
21:22:11.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:11.193774 11012 fs_manager.cc:362] Metadata directory not provided
21:22:11.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:11.194046 11012 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:22:11.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:11.194587 11012 server_base.cc:908] This appears to be a new deployment of Kudu; creating new FS layout
21:22:11.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:11.200788 11012 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/master-0/data/instance:
21:22:11.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "04f29f2996a546f79797e6e6920a700f"
21:22:11.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-01-28 21:22:11 on dist-test-slave-hq3s"
21:22:11.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:11.202301 11012 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/master-0/wal/instance:
21:22:11.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "04f29f2996a546f79797e6e6920a700f"
21:22:11.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-01-28 21:22:11 on dist-test-slave-hq3s"
21:22:11.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:11.212373 11012 fs_manager.cc:696] Time spent creating directory manager: real 0.009s user 0.009s sys 0.002s
21:22:11.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:11.220044 11030 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:22:11.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:11.221967 11012 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.002s sys 0.002s
21:22:11.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:11.222590 11012 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/master-0/data,/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/master-0/wal
21:22:11.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "04f29f2996a546f79797e6e6920a700f"
21:22:11.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-01-28 21:22:11 on dist-test-slave-hq3s"
21:22:11.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:11.223052 11012 fs_report.cc:389] FS layout report
21:22:11.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:22:11.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/master-0/wal
21:22:11.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/master-0/wal
21:22:11.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/master-0/data/data
21:22:11.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:22:11.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:22:11.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:22:11.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:22:11.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:22:11.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:22:11.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:22:11.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:22:11.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:22:11.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:22:11.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:22:11.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:11.304937 11012 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:22:11.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:11.310424 11012 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:22:11.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:11.310904 11012 kserver.cc:163] Server-wide thread pool size limit: 3276
21:22:11.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:11.378165 11012 rpc_server.cc:307] RPC server started. Bound to: 127.10.186.62:46125
21:22:11.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:11.378232 11081 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.10.186.62:46125 every 8 connection(s)
21:22:11.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:11.380810 11012 server_base.cc:1192] Dumped server information to /tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/master-0/data/info.pb
21:22:11.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:11.385048 10984 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskp7EThw/build/tsan/bin/kudu as pid 11012
21:22:11.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:11.385514 10984 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/master-0/wal/instance
21:22:11.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:11.387168 11082 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
21:22:11.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.10.186.61@KRBTEST.COM; defaulting to no policy
21:22:11.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:11.398651 11082 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.186.62" port: 46125 } has no permanent_uuid. Determining permanent_uuid...
21:22:11.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:22:11.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.10.186.61@KRBTEST.COM" created.
21:22:11.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 28 21:22:11 dist-test-slave-hq3s krb5kdc[10996](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1769635331, etypes {rep=17 tkt=17 ses=17}, kudu/127.10.186.62@KRBTEST.COM for kudu/127.10.186.62@KRBTEST.COM
21:22:11.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:22:11.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.10.186.61 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/krb5kdc/kudu.keytab.
21:22:11.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.10.186.61 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/krb5kdc/kudu.keytab.
21:22:11.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:11.436923 11082 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.186.61" port: 45819 } has no permanent_uuid. Determining permanent_uuid...
21:22:11.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:11.439044 11033 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.10.186.61:45819: connect: Connection refused (error 111)
21:22:11.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.10.186.61@KRBTEST.COM; defaulting to no policy
21:22:11.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:22:11.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.10.186.61@KRBTEST.COM" created.
21:22:11.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:11.442934 11082 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.10.186.61:45819: Network error: Client connection negotiation failed: client connection to 127.10.186.61:45819: connect: Connection refused (error 111)
21:22:11.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:22:11.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.10.186.61 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/krb5kdc/kudu.keytab.
21:22:11.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.10.186.61 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/krb5kdc/kudu.keytab.
21:22:11.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:11.463014 10984 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskp7EThw/build/tsan/bin/kudu
21:22:11.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskp7EThw/build/tsan/bin/kudu
21:22:11.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/master-1/wal
21:22:11.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/master-1/data
21:22:11.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:22:11.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:22:11.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:22:11.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:22:11.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:22:11.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:22:11.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/master-1/logs
21:22:11.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/master-1/data/info.pb
21:22:11.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:22:11.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:22:11.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:22:11.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:22:11.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:22:11.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:22:11.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:22:11.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:22:11.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:22:11.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:22:11.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:22:11.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:22:11.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.186.61:45819
21:22:11.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.186.61
21:22:11.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:22:11.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/krb5kdc/kudu.keytab
21:22:11.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.10.186.61
21:22:11.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:22:11.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
21:22:11.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
21:22:11.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
21:22:11.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.186.20:34545
21:22:11.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:22:11.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:22:11.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:22:11.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:22:11.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.186.62:46125,127.10.186.61:45819,127.10.186.60:38107
21:22:11.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:22:11.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34709/jwks.json
21:22:11.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
21:22:11.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
21:22:11.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:11.492437 11082 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.186.61" port: 45819 } attempt: 1
21:22:11.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:11.496281 11082 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.10.186.61:45819: Network error: Client connection negotiation failed: client connection to 127.10.186.61:45819: connect: Connection refused (error 111)
21:22:11.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:11.564675 11082 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.186.61" port: 45819 } attempt: 2
21:22:11.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:11.568307 11082 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.10.186.61:45819: Network error: Client connection negotiation failed: client connection to 127.10.186.61:45819: connect: Connection refused (error 111)
21:22:11.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:11.659672 11082 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.186.61" port: 45819 } attempt: 3
21:22:11.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:11.663928 11082 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.10.186.61:45819: Network error: Client connection negotiation failed: client connection to 127.10.186.61:45819: connect: Connection refused (error 111)
21:22:11.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:11.755957 11091 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:22:11.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:11.756493 11091 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:22:11.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:11.757095 11091 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:22:11.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:11.788388 11091 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
21:22:11.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:11.788648 11091 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:22:11.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:11.788933 11091 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:22:11.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:11.789252 11091 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:22:11.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:11.789588 11091 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:22:11.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:11.789888 11091 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:22:11.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:11.790056 11091 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34709/jwks.json
21:22:11.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:11.807366 11082 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.186.61" port: 45819 } attempt: 4
21:22:11.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:11.811061 11082 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.10.186.61:45819: Network error: Client connection negotiation failed: client connection to 127.10.186.61:45819: connect: Connection refused (error 111)
21:22:11.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:11.828891 11091 master_runner.cc:386] Master server non-default flags:
21:22:11.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:22:11.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.186.20:34545
21:22:11.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:22:11.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:22:11.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/master-1/data
21:22:11.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/master-1/wal
21:22:11.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
21:22:11.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:22:11.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.186.62:46125,127.10.186.61:45819,127.10.186.60:38107
21:22:11.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/krb5kdc/kudu.keytab
21:22:11.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.10.186.61
21:22:11.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:22:11.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:22:11.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:22:11.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.186.61:45819
21:22:11.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:22:11.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:22:11.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:22:11.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34709/jwks.json
21:22:11.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:22:11.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
21:22:11.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
21:22:11.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:22:11.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:22:11.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/master-1/data/info.pb
21:22:11.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.186.61
21:22:11.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:22:11.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:22:11.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:22:11.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:22:11.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:22:11.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:22:11.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/master-1/logs
21:22:11.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:22:11.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:22:11.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:22:11.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:22:11.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:22:11.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3722ed49c19ce54c7feae2fa944e32b1d4ed8d2d
21:22:11.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:22:11.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 28 Jan 2026 21:02:00 UTC on 24a791456cd2
21:22:11.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10237
21:22:11.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:22:11.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:11.830075 11091 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:22:11.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:11.831004 11091 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:22:11.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:11.841847 11097 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:22:12.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:12.110459 11082 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.186.61" port: 45819 } attempt: 5
21:22:12.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:12.117844 11082 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.10.186.61:45819: Network error: Client connection negotiation failed: client connection to 127.10.186.61:45819: connect: Connection refused (error 111)
21:22:12.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:12.665300 11082 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.186.61" port: 45819 } attempt: 6
21:22:12.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:12.699977 11082 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.10.186.61:45819: Network error: Client connection negotiation failed: client connection to 127.10.186.61:45819: connect: Connection refused (error 111)
21:22:13.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:11.843145 11098 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:22:13.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:13.110929 11100 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:22:13.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:13.112849 11099 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1265 milliseconds
21:22:13.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:13.113760 11091 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.271s user 0.502s sys 0.761s
21:22:13.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:13.114027 11091 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.272s user 0.502s sys 0.761s
21:22:13.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:13.114275 11091 server_base.cc:1055] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:22:13.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:13.115329 11091 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:22:13.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:13.117934 11091 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:22:13.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:13.119529 11091 hybrid_clock.cc:648] HybridClock initialized: now 1769635333119477 us; error 47 us; skew 500 ppm
21:22:13.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:13.120694 11091 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:22:13.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 28 21:22:13 dist-test-slave-hq3s krb5kdc[10996](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1769635333, etypes {rep=17 tkt=17 ses=17}, kudu/127.10.186.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:22:13.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:13.128772 11091 init.cc:377] Logged in from keytab as kudu/127.10.186.61@KRBTEST.COM (short username kudu)
21:22:13.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:13.142887 11091 webserver.cc:492] Webserver started at http://127.10.186.61:33271/ using document root <none> and password file <none>
21:22:13.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:13.143996 11091 fs_manager.cc:362] Metadata directory not provided
21:22:13.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:13.144230 11091 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:22:13.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:13.144695 11091 server_base.cc:908] This appears to be a new deployment of Kudu; creating new FS layout
21:22:13.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:13.149070 11091 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/master-1/data/instance:
21:22:13.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5ada6348d26f452da1cd3c3f9f9294f4"
21:22:13.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-01-28 21:22:13 on dist-test-slave-hq3s"
21:22:13.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:13.150542 11091 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/master-1/wal/instance:
21:22:13.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5ada6348d26f452da1cd3c3f9f9294f4"
21:22:13.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-01-28 21:22:13 on dist-test-slave-hq3s"
21:22:13.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:13.159402 11091 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.005s sys 0.001s
21:22:13.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:13.165086 11114 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:22:13.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:13.166288 11091 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.002s
21:22:13.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:13.166643 11091 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/master-1/data,/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/master-1/wal
21:22:13.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5ada6348d26f452da1cd3c3f9f9294f4"
21:22:13.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-01-28 21:22:13 on dist-test-slave-hq3s"
21:22:13.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:13.166963 11091 fs_report.cc:389] FS layout report
21:22:13.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:22:13.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/master-1/wal
21:22:13.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/master-1/wal
21:22:13.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/master-1/data/data
21:22:13.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:22:13.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:22:13.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:22:13.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:22:13.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:22:13.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:22:13.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:22:13.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:22:13.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:22:13.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:22:13.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:22:13.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:13.242493 11091 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:22:13.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:13.251451 11091 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:22:13.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:13.251919 11091 kserver.cc:163] Server-wide thread pool size limit: 3276
21:22:13.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:13.321566 11091 rpc_server.cc:307] RPC server started. Bound to: 127.10.186.61:45819
21:22:13.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:13.321651 11166 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.10.186.61:45819 every 8 connection(s)
21:22:13.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:13.324132 11091 server_base.cc:1192] Dumped server information to /tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/master-1/data/info.pb
21:22:13.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:13.328974 10984 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskp7EThw/build/tsan/bin/kudu as pid 11091
21:22:13.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:13.329435 10984 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/master-1/wal/instance
21:22:13.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:13.330996 11167 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
21:22:13.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.10.186.60@KRBTEST.COM; defaulting to no policy
21:22:13.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:13.343139 11167 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.186.62" port: 46125 } has no permanent_uuid. Determining permanent_uuid...
21:22:13.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:22:13.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.10.186.60@KRBTEST.COM" created.
21:22:13.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 28 21:22:13 dist-test-slave-hq3s krb5kdc[10996](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1769635333, etypes {rep=17 tkt=17 ses=17}, kudu/127.10.186.61@KRBTEST.COM for kudu/127.10.186.62@KRBTEST.COM
21:22:13.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:22:13.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.10.186.60 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/krb5kdc/kudu.keytab.
21:22:13.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.10.186.60 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/krb5kdc/kudu.keytab.
21:22:13.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:13.379704 11167 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.186.61" port: 45819 } has no permanent_uuid. Determining permanent_uuid...
21:22:13.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.10.186.60@KRBTEST.COM; defaulting to no policy
21:22:13.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:22:13.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.10.186.60@KRBTEST.COM" created.
21:22:13.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 28 21:22:13 dist-test-slave-hq3s krb5kdc[10996](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1769635333, etypes {rep=17 tkt=17 ses=17}, kudu/127.10.186.61@KRBTEST.COM for kudu/127.10.186.61@KRBTEST.COM
21:22:13.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:13.410072 11167 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.186.60" port: 38107 } has no permanent_uuid. Determining permanent_uuid...
21:22:13.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:22:13.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.10.186.60 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/krb5kdc/kudu.keytab.
21:22:13.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.10.186.60 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/krb5kdc/kudu.keytab.
21:22:13.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:13.412161 11118 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.10.186.60:38107: connect: Connection refused (error 111)
21:22:13.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:13.414247 10984 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskp7EThw/build/tsan/bin/kudu
21:22:13.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskp7EThw/build/tsan/bin/kudu
21:22:13.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/master-2/wal
21:22:13.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/master-2/data
21:22:13.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:22:13.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:22:13.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:22:13.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:22:13.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:22:13.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:22:13.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/master-2/logs
21:22:13.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/master-2/data/info.pb
21:22:13.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:22:13.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:22:13.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:22:13.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:22:13.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:22:13.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:22:13.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:22:13.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:22:13.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:22:13.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:22:13.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:22:13.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:22:13.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.186.60:38107
21:22:13.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.186.60
21:22:13.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:22:13.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/krb5kdc/kudu.keytab
21:22:13.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.10.186.60
21:22:13.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:22:13.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
21:22:13.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
21:22:13.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
21:22:13.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.186.20:34545
21:22:13.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:22:13.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:22:13.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:22:13.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:22:13.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.186.62:46125,127.10.186.61:45819,127.10.186.60:38107
21:22:13.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:22:13.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34709/jwks.json
21:22:13.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
21:22:13.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
21:22:13.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:13.415995 11167 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.10.186.60:38107: Network error: Client connection negotiation failed: client connection to 127.10.186.60:38107: connect: Connection refused (error 111)
21:22:13.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:13.465528 11167 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.186.60" port: 38107 } attempt: 1
21:22:13.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:13.469368 11167 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.10.186.60:38107: Network error: Client connection negotiation failed: client connection to 127.10.186.60:38107: connect: Connection refused (error 111)
21:22:13.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:13.537731 11167 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.186.60" port: 38107 } attempt: 2
21:22:13.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:13.541355 11167 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.10.186.60:38107: Network error: Client connection negotiation failed: client connection to 127.10.186.60:38107: connect: Connection refused (error 111)
21:22:13.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:13.632789 11167 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.186.60" port: 38107 } attempt: 3
21:22:13.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:13.636689 11167 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.10.186.60:38107: Network error: Client connection negotiation failed: client connection to 127.10.186.60:38107: connect: Connection refused (error 111)
21:22:13.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:13.707607 11177 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:22:13.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:13.708207 11177 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:22:13.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:13.708916 11177 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:22:13.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:13.740757 11177 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
21:22:13.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:13.741024 11177 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:22:13.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:13.741386 11177 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:22:13.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:13.741664 11177 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:22:13.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:13.741925 11177 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:22:13.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:13.742195 11177 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:22:13.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:13.742394 11177 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34709/jwks.json
21:22:13.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:13.760387 11082 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.186.61" port: 45819 } attempt: 7
21:22:13.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 28 21:22:13 dist-test-slave-hq3s krb5kdc[10996](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1769635331, etypes {rep=17 tkt=17 ses=17}, kudu/127.10.186.62@KRBTEST.COM for kudu/127.10.186.61@KRBTEST.COM
21:22:13.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:13.780020 11167 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.186.60" port: 38107 } attempt: 4
21:22:13.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:13.781762 11082 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.186.60" port: 38107 } has no permanent_uuid. Determining permanent_uuid...
21:22:13.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:13.783061 11177 master_runner.cc:386] Master server non-default flags:
21:22:13.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:22:13.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.186.20:34545
21:22:13.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:22:13.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:22:13.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/master-2/data
21:22:13.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/master-2/wal
21:22:13.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
21:22:13.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:22:13.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.186.62:46125,127.10.186.61:45819,127.10.186.60:38107
21:22:13.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/krb5kdc/kudu.keytab
21:22:13.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.10.186.60
21:22:13.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:22:13.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:22:13.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:22:13.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.186.60:38107
21:22:13.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:22:13.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:22:13.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:22:13.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34709/jwks.json
21:22:13.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:22:13.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
21:22:13.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
21:22:13.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:22:13.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:22:13.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/master-2/data/info.pb
21:22:13.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.186.60
21:22:13.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:22:13.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:22:13.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:22:13.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:22:13.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:22:13.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:22:13.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/master-2/logs
21:22:13.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:22:13.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:22:13.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:22:13.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:22:13.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:22:13.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3722ed49c19ce54c7feae2fa944e32b1d4ed8d2d
21:22:13.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:22:13.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 28 Jan 2026 21:02:00 UTC on 24a791456cd2
21:22:13.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10237
21:22:13.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:22:13.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:13.784061 11167 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.10.186.60:38107: Network error: Client connection negotiation failed: client connection to 127.10.186.60:38107: connect: Connection refused (error 111)
21:22:13.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:13.784598 11177 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:22:13.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:13.785602 11177 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:22:13.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:13.788542 11082 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.10.186.60:38107: Network error: Client connection negotiation failed: client connection to 127.10.186.60:38107: connect: Connection refused (error 111)
21:22:13.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:13.796856 11185 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:22:13.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:13.846999 11082 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.186.60" port: 38107 } attempt: 1
21:22:13.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:13.852908 11082 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.10.186.60:38107: Network error: Client connection negotiation failed: client connection to 127.10.186.60:38107: connect: Connection refused (error 111)
21:22:13.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:13.934376 11082 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.186.60" port: 38107 } attempt: 2
21:22:13.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:13.940356 11082 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.10.186.60:38107: Network error: Client connection negotiation failed: client connection to 127.10.186.60:38107: connect: Connection refused (error 111)
21:22:14.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:14.025832 11082 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.186.60" port: 38107 } attempt: 3
21:22:14.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:14.031380 11082 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.10.186.60:38107: Network error: Client connection negotiation failed: client connection to 127.10.186.60:38107: connect: Connection refused (error 111)
21:22:14.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:14.083482 11167 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.186.60" port: 38107 } attempt: 5
21:22:14.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:14.089761 11167 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.10.186.60:38107: Network error: Client connection negotiation failed: client connection to 127.10.186.60:38107: connect: Connection refused (error 111)
21:22:14.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:14.171903 11082 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.186.60" port: 38107 } attempt: 4
21:22:14.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:14.178256 11082 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.10.186.60:38107: Network error: Client connection negotiation failed: client connection to 127.10.186.60:38107: connect: Connection refused (error 111)
21:22:14.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:14.461735 11082 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.186.60" port: 38107 } attempt: 5
21:22:14.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:14.467850 11082 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.10.186.60:38107: Network error: Client connection negotiation failed: client connection to 127.10.186.60:38107: connect: Connection refused (error 111)
21:22:14.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:14.637240 11167 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.186.60" port: 38107 } attempt: 6
21:22:14.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:14.675156 11167 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.10.186.60:38107: Network error: Client connection negotiation failed: client connection to 127.10.186.60:38107: connect: Connection refused (error 111)
21:22:15.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:15.020299 11082 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.186.60" port: 38107 } attempt: 6
21:22:15.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:15.067123 11082 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.10.186.60:38107: Network error: Client connection negotiation failed: client connection to 127.10.186.60:38107: connect: Connection refused (error 111)
21:22:15.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:15.204595 11184 debug-util.cc:398] Leaking SignalData structure 0x7b0800034ca0 after lost signal to thread 11177
21:22:15.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:15.302649 11184 kernel_stack_watchdog.cc:198] Thread 11177 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 405ms:
21:22:15.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
21:22:15.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
21:22:15.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
21:22:15.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
21:22:15.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:15.302908 11177 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.506s user 0.480s sys 0.998s
21:22:15.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:15.303296 11177 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.507s user 0.480s sys 0.998s
21:22:15.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:13.799276 11186 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:22:15.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:15.304122 11187 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1506 milliseconds
21:22:15.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:15.305131 11188 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:22:15.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:15.305105 11177 server_base.cc:1055] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:22:15.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:15.309221 11177 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:22:15.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:15.311645 11177 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:22:15.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:15.312987 11177 hybrid_clock.cc:648] HybridClock initialized: now 1769635335312943 us; error 46 us; skew 500 ppm
21:22:15.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:15.313920 11177 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:22:15.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 28 21:22:15 dist-test-slave-hq3s krb5kdc[10996](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1769635335, etypes {rep=17 tkt=17 ses=17}, kudu/127.10.186.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:22:15.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:15.319654 11177 init.cc:377] Logged in from keytab as kudu/127.10.186.60@KRBTEST.COM (short username kudu)
21:22:15.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:15.326700 11177 webserver.cc:492] Webserver started at http://127.10.186.60:36489/ using document root <none> and password file <none>
21:22:15.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:15.327639 11177 fs_manager.cc:362] Metadata directory not provided
21:22:15.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:15.327850 11177 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:22:15.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:15.328241 11177 server_base.cc:908] This appears to be a new deployment of Kudu; creating new FS layout
21:22:15.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:15.332412 11177 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/master-2/data/instance:
21:22:15.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f7480f49edbd4746ac36c92674ce0524"
21:22:15.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-01-28 21:22:15 on dist-test-slave-hq3s"
21:22:15.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:15.333570 11177 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/master-2/wal/instance:
21:22:15.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f7480f49edbd4746ac36c92674ce0524"
21:22:15.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-01-28 21:22:15 on dist-test-slave-hq3s"
21:22:15.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:15.340696 11177 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.002s sys 0.006s
21:22:15.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:15.346040 11201 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:22:15.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:15.347046 11177 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.005s sys 0.001s
21:22:15.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:15.347324 11177 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/master-2/data,/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/master-2/wal
21:22:15.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f7480f49edbd4746ac36c92674ce0524"
21:22:15.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-01-28 21:22:15 on dist-test-slave-hq3s"
21:22:15.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:15.347666 11177 fs_report.cc:389] FS layout report
21:22:15.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:22:15.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/master-2/wal
21:22:15.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/master-2/wal
21:22:15.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/master-2/data/data
21:22:15.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:22:15.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:22:15.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:22:15.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:22:15.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:22:15.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:22:15.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:22:15.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:22:15.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:22:15.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:22:15.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:22:15.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:15.393421 11177 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:22:15.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:15.398615 11177 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:22:15.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:15.399017 11177 kserver.cc:163] Server-wide thread pool size limit: 3276
21:22:15.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:15.474877 11177 rpc_server.cc:307] RPC server started. Bound to: 127.10.186.60:38107
21:22:15.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:15.474951 11253 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.10.186.60:38107 every 8 connection(s)
21:22:15.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:15.477628 11177 server_base.cc:1192] Dumped server information to /tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/master-2/data/info.pb
21:22:15.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:15.483738 11254 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
21:22:15.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:15.487228 10984 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskp7EThw/build/tsan/bin/kudu as pid 11177
21:22:15.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:15.487753 10984 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/master-2/wal/instance
21:22:15.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:15.496618 11254 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.186.62" port: 46125 } has no permanent_uuid. Determining permanent_uuid...
21:22:15.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 28 21:22:15 dist-test-slave-hq3s krb5kdc[10996](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1769635335, etypes {rep=17 tkt=17 ses=17}, kudu/127.10.186.60@KRBTEST.COM for kudu/127.10.186.62@KRBTEST.COM
21:22:15.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 28 21:22:15 dist-test-slave-hq3s krb5kdc[10996](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1769635329, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.10.186.60@KRBTEST.COM
21:22:15.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:15.528334 11254 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.186.61" port: 45819 } has no permanent_uuid. Determining permanent_uuid...
21:22:15.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 28 21:22:15 dist-test-slave-hq3s krb5kdc[10996](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1769635335, etypes {rep=17 tkt=17 ses=17}, kudu/127.10.186.60@KRBTEST.COM for kudu/127.10.186.61@KRBTEST.COM
21:22:15.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 28 21:22:15 dist-test-slave-hq3s krb5kdc[10996](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1769635329, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.10.186.61@KRBTEST.COM
21:22:15.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:15.552454 11254 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.186.60" port: 38107 } has no permanent_uuid. Determining permanent_uuid...
21:22:15.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 28 21:22:15 dist-test-slave-hq3s krb5kdc[10996](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1769635335, etypes {rep=17 tkt=17 ses=17}, kudu/127.10.186.60@KRBTEST.COM for kudu/127.10.186.60@KRBTEST.COM
21:22:15.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 28 21:22:15 dist-test-slave-hq3s krb5kdc[10996](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1769635329, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.10.186.62@KRBTEST.COM
21:22:15.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:15.579245 11254 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f7480f49edbd4746ac36c92674ce0524: Bootstrap starting.
21:22:15.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:15.584887 10984 external_mini_cluster.cc:949] 0 TS(s) registered with all masters
21:22:15.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:15.585817 11254 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P f7480f49edbd4746ac36c92674ce0524: Neither blocks nor log segments found. Creating new log.
21:22:15.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:15.587430 11254 log.cc:826] T 00000000000000000000000000000000 P f7480f49edbd4746ac36c92674ce0524: Log is configured to *not* fsync() on all Append() calls
21:22:15.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:15.591861 11254 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f7480f49edbd4746ac36c92674ce0524: No bootstrap required, opened a new log
21:22:15.592 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:22:15.592 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
21:22:15.593 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
21:22:15.593 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
21:22:15.594 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.10.186.62"
port: 46125
}
bound_http_address {
host: "127.10.186.62"
port: 37867
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.10.186.61"
port: 45819
}
bound_http_address {
host: "127.10.186.61"
port: 33271
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.10.186.60"
port: 38107
}
bound_http_address {
host: "127.10.186.60"
port: 36489
}
}
}
21:22:15.594 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
21:22:15.594 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:22:15.594 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
21:22:15.594 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
21:22:15.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:15.611191 11254 raft_consensus.cc:359] T 00000000000000000000000000000000 P f7480f49edbd4746ac36c92674ce0524 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "04f29f2996a546f79797e6e6920a700f" member_type: VOTER last_known_addr { host: "127.10.186.62" port: 46125 } } peers { permanent_uuid: "5ada6348d26f452da1cd3c3f9f9294f4" member_type: VOTER last_known_addr { host: "127.10.186.61" port: 45819 } } peers { permanent_uuid: "f7480f49edbd4746ac36c92674ce0524" member_type: VOTER last_known_addr { host: "127.10.186.60" port: 38107 } }
21:22:15.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:15.611927 11254 raft_consensus.cc:385] T 00000000000000000000000000000000 P f7480f49edbd4746ac36c92674ce0524 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:22:15.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:15.612196 11254 raft_consensus.cc:740] T 00000000000000000000000000000000 P f7480f49edbd4746ac36c92674ce0524 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f7480f49edbd4746ac36c92674ce0524, State: Initialized, Role: FOLLOWER
21:22:15.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:15.613016 11254 consensus_queue.cc:260] T 00000000000000000000000000000000 P f7480f49edbd4746ac36c92674ce0524 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "04f29f2996a546f79797e6e6920a700f" member_type: VOTER last_known_addr { host: "127.10.186.62" port: 46125 } } peers { permanent_uuid: "5ada6348d26f452da1cd3c3f9f9294f4" member_type: VOTER last_known_addr { host: "127.10.186.61" port: 45819 } } peers { permanent_uuid: "f7480f49edbd4746ac36c92674ce0524" member_type: VOTER last_known_addr { host: "127.10.186.60" port: 38107 } }
21:22:15.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:15.615717 11265 sys_catalog.cc:455] T 00000000000000000000000000000000 P f7480f49edbd4746ac36c92674ce0524 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "04f29f2996a546f79797e6e6920a700f" member_type: VOTER last_known_addr { host: "127.10.186.62" port: 46125 } } peers { permanent_uuid: "5ada6348d26f452da1cd3c3f9f9294f4" member_type: VOTER last_known_addr { host: "127.10.186.61" port: 45819 } } peers { permanent_uuid: "f7480f49edbd4746ac36c92674ce0524" member_type: VOTER last_known_addr { host: "127.10.186.60" port: 38107 } } }
21:22:15.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:15.616494 11265 sys_catalog.cc:458] T 00000000000000000000000000000000 P f7480f49edbd4746ac36c92674ce0524 [sys.catalog]: This master's current role is: FOLLOWER
21:22:15.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:15.617606 11254 sys_catalog.cc:565] T 00000000000000000000000000000000 P f7480f49edbd4746ac36c92674ce0524 [sys.catalog]: configured and running, proceeding with master startup.
21:22:15.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 28 21:22:15 dist-test-slave-hq3s krb5kdc[10996](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1769635335, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:22:15.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
21:22:15.626 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:22:15.626 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/krb5kdc/krb5cc
21:22:15.627 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
21:22:15.628 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/krb5kdc/krb5cc
21:22:15.628 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
21:22:15.629 [DEBUG - Time-limited test] (SecurityContext.java:158) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
21:22:15.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:15.641916 11276 catalog_manager.cc:1568] T 00000000000000000000000000000000 P f7480f49edbd4746ac36c92674ce0524: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:22:15.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:15.642238 11276 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:22:15.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:15.653772 11207 tablet.cc:2378] T 00000000000000000000000000000000 P f7480f49edbd4746ac36c92674ce0524: Can't schedule compaction. Clean time has not been advanced past its initial value.
21:22:15.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:15.638211 11177 master_runner.cc:186] Error getting master registration for 127.10.186.62:46125: OK, instance_id { permanent_uuid: "04f29f2996a546f79797e6e6920a700f" instance_seqno: 1769635331359834 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
21:22:15.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:15.658355 11177 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
21:22:15.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:15.735682 11167 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.186.60" port: 38107 } attempt: 7
21:22:15.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 28 21:22:15 dist-test-slave-hq3s krb5kdc[10996](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1769635333, etypes {rep=17 tkt=17 ses=17}, kudu/127.10.186.61@KRBTEST.COM for kudu/127.10.186.60@KRBTEST.COM
21:22:15.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:15.765357 11167 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5ada6348d26f452da1cd3c3f9f9294f4: Bootstrap starting.
21:22:15.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:15.770881 11167 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 5ada6348d26f452da1cd3c3f9f9294f4: Neither blocks nor log segments found. Creating new log.
21:22:15.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:15.773306 11167 log.cc:826] T 00000000000000000000000000000000 P 5ada6348d26f452da1cd3c3f9f9294f4: Log is configured to *not* fsync() on all Append() calls
21:22:15.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:15.777935 11167 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5ada6348d26f452da1cd3c3f9f9294f4: No bootstrap required, opened a new log
21:22:15.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:15.795753 11167 raft_consensus.cc:359] T 00000000000000000000000000000000 P 5ada6348d26f452da1cd3c3f9f9294f4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "04f29f2996a546f79797e6e6920a700f" member_type: VOTER last_known_addr { host: "127.10.186.62" port: 46125 } } peers { permanent_uuid: "5ada6348d26f452da1cd3c3f9f9294f4" member_type: VOTER last_known_addr { host: "127.10.186.61" port: 45819 } } peers { permanent_uuid: "f7480f49edbd4746ac36c92674ce0524" member_type: VOTER last_known_addr { host: "127.10.186.60" port: 38107 } }
21:22:15.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:15.796528 11167 raft_consensus.cc:385] T 00000000000000000000000000000000 P 5ada6348d26f452da1cd3c3f9f9294f4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:22:15.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:15.796743 11167 raft_consensus.cc:740] T 00000000000000000000000000000000 P 5ada6348d26f452da1cd3c3f9f9294f4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5ada6348d26f452da1cd3c3f9f9294f4, State: Initialized, Role: FOLLOWER
21:22:15.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:15.797428 11167 consensus_queue.cc:260] T 00000000000000000000000000000000 P 5ada6348d26f452da1cd3c3f9f9294f4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "04f29f2996a546f79797e6e6920a700f" member_type: VOTER last_known_addr { host: "127.10.186.62" port: 46125 } } peers { permanent_uuid: "5ada6348d26f452da1cd3c3f9f9294f4" member_type: VOTER last_known_addr { host: "127.10.186.61" port: 45819 } } peers { permanent_uuid: "f7480f49edbd4746ac36c92674ce0524" member_type: VOTER last_known_addr { host: "127.10.186.60" port: 38107 } }
21:22:15.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:15.800168 11280 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5ada6348d26f452da1cd3c3f9f9294f4 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "04f29f2996a546f79797e6e6920a700f" member_type: VOTER last_known_addr { host: "127.10.186.62" port: 46125 } } peers { permanent_uuid: "5ada6348d26f452da1cd3c3f9f9294f4" member_type: VOTER last_known_addr { host: "127.10.186.61" port: 45819 } } peers { permanent_uuid: "f7480f49edbd4746ac36c92674ce0524" member_type: VOTER last_known_addr { host: "127.10.186.60" port: 38107 } } }
21:22:15.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:15.800976 11280 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5ada6348d26f452da1cd3c3f9f9294f4 [sys.catalog]: This master's current role is: FOLLOWER
21:22:15.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:15.801937 11167 sys_catalog.cc:565] T 00000000000000000000000000000000 P 5ada6348d26f452da1cd3c3f9f9294f4 [sys.catalog]: configured and running, proceeding with master startup.
21:22:15.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:15.829985 11291 catalog_manager.cc:1568] T 00000000000000000000000000000000 P 5ada6348d26f452da1cd3c3f9f9294f4: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:22:15.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:15.830328 11291 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:22:15.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:15.826316 11091 master_runner.cc:186] Error getting master registration for 127.10.186.62:46125: OK, instance_id { permanent_uuid: "04f29f2996a546f79797e6e6920a700f" instance_seqno: 1769635331359834 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
21:22:15.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:15.856886 11091 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
21:22:16.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:16.009346 11121 tablet.cc:2378] T 00000000000000000000000000000000 P 5ada6348d26f452da1cd3c3f9f9294f4: Can't schedule compaction. Clean time has not been advanced past its initial value.
21:22:16.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:16.029160 11265 raft_consensus.cc:493] T 00000000000000000000000000000000 P f7480f49edbd4746ac36c92674ce0524 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:22:16.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:16.029745 11265 raft_consensus.cc:515] T 00000000000000000000000000000000 P f7480f49edbd4746ac36c92674ce0524 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "04f29f2996a546f79797e6e6920a700f" member_type: VOTER last_known_addr { host: "127.10.186.62" port: 46125 } } peers { permanent_uuid: "5ada6348d26f452da1cd3c3f9f9294f4" member_type: VOTER last_known_addr { host: "127.10.186.61" port: 45819 } } peers { permanent_uuid: "f7480f49edbd4746ac36c92674ce0524" member_type: VOTER last_known_addr { host: "127.10.186.60" port: 38107 } }
21:22:16.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:16.032274 11265 leader_election.cc:290] T 00000000000000000000000000000000 P f7480f49edbd4746ac36c92674ce0524 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 04f29f2996a546f79797e6e6920a700f (127.10.186.62:46125), 5ada6348d26f452da1cd3c3f9f9294f4 (127.10.186.61:45819)
21:22:16.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:16.033759 11142 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "f7480f49edbd4746ac36c92674ce0524" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5ada6348d26f452da1cd3c3f9f9294f4" is_pre_election: true
21:22:16.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:16.034662 11142 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 5ada6348d26f452da1cd3c3f9f9294f4 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate f7480f49edbd4746ac36c92674ce0524 in term 0.
21:22:16.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:16.036146 11204 leader_election.cc:304] T 00000000000000000000000000000000 P f7480f49edbd4746ac36c92674ce0524 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 5ada6348d26f452da1cd3c3f9f9294f4, f7480f49edbd4746ac36c92674ce0524; no voters:
21:22:16.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:16.037088 11265 raft_consensus.cc:2804] T 00000000000000000000000000000000 P f7480f49edbd4746ac36c92674ce0524 [term 0 FOLLOWER]: Leader pre-election won for term 1
21:22:16.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:16.037423 11265 raft_consensus.cc:493] T 00000000000000000000000000000000 P f7480f49edbd4746ac36c92674ce0524 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
21:22:16.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:16.037750 11265 raft_consensus.cc:3060] T 00000000000000000000000000000000 P f7480f49edbd4746ac36c92674ce0524 [term 0 FOLLOWER]: Advancing to term 1
21:22:16.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:16.044399 11265 raft_consensus.cc:515] T 00000000000000000000000000000000 P f7480f49edbd4746ac36c92674ce0524 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "04f29f2996a546f79797e6e6920a700f" member_type: VOTER last_known_addr { host: "127.10.186.62" port: 46125 } } peers { permanent_uuid: "5ada6348d26f452da1cd3c3f9f9294f4" member_type: VOTER last_known_addr { host: "127.10.186.61" port: 45819 } } peers { permanent_uuid: "f7480f49edbd4746ac36c92674ce0524" member_type: VOTER last_known_addr { host: "127.10.186.60" port: 38107 } }
21:22:16.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:16.047016 11265 leader_election.cc:290] T 00000000000000000000000000000000 P f7480f49edbd4746ac36c92674ce0524 [CANDIDATE]: Term 1 election: Requested vote from peers 04f29f2996a546f79797e6e6920a700f (127.10.186.62:46125), 5ada6348d26f452da1cd3c3f9f9294f4 (127.10.186.61:45819)
21:22:16.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:16.058708 11142 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "f7480f49edbd4746ac36c92674ce0524" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5ada6348d26f452da1cd3c3f9f9294f4"
21:22:16.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:16.059300 11142 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 5ada6348d26f452da1cd3c3f9f9294f4 [term 0 FOLLOWER]: Advancing to term 1
21:22:16.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:16.066547 11142 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 5ada6348d26f452da1cd3c3f9f9294f4 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate f7480f49edbd4746ac36c92674ce0524 in term 1.
21:22:16.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:16.067952 11204 leader_election.cc:304] T 00000000000000000000000000000000 P f7480f49edbd4746ac36c92674ce0524 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 5ada6348d26f452da1cd3c3f9f9294f4, f7480f49edbd4746ac36c92674ce0524; no voters:
21:22:16.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:16.069116 11265 raft_consensus.cc:2804] T 00000000000000000000000000000000 P f7480f49edbd4746ac36c92674ce0524 [term 1 FOLLOWER]: Leader election won for term 1
21:22:16.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:16.071746 11265 raft_consensus.cc:697] T 00000000000000000000000000000000 P f7480f49edbd4746ac36c92674ce0524 [term 1 LEADER]: Becoming Leader. State: Replica: f7480f49edbd4746ac36c92674ce0524, State: Running, Role: LEADER
21:22:16.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:16.075307 11265 consensus_queue.cc:237] T 00000000000000000000000000000000 P f7480f49edbd4746ac36c92674ce0524 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "04f29f2996a546f79797e6e6920a700f" member_type: VOTER last_known_addr { host: "127.10.186.62" port: 46125 } } peers { permanent_uuid: "5ada6348d26f452da1cd3c3f9f9294f4" member_type: VOTER last_known_addr { host: "127.10.186.61" port: 45819 } } peers { permanent_uuid: "f7480f49edbd4746ac36c92674ce0524" member_type: VOTER last_known_addr { host: "127.10.186.60" port: 38107 } }
21:22:16.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:16.092257 11293 sys_catalog.cc:455] T 00000000000000000000000000000000 P f7480f49edbd4746ac36c92674ce0524 [sys.catalog]: SysCatalogTable state changed. Reason: New leader f7480f49edbd4746ac36c92674ce0524. Latest consensus state: current_term: 1 leader_uuid: "f7480f49edbd4746ac36c92674ce0524" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "04f29f2996a546f79797e6e6920a700f" member_type: VOTER last_known_addr { host: "127.10.186.62" port: 46125 } } peers { permanent_uuid: "5ada6348d26f452da1cd3c3f9f9294f4" member_type: VOTER last_known_addr { host: "127.10.186.61" port: 45819 } } peers { permanent_uuid: "f7480f49edbd4746ac36c92674ce0524" member_type: VOTER last_known_addr { host: "127.10.186.60" port: 38107 } } }
21:22:16.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:16.092991 11293 sys_catalog.cc:458] T 00000000000000000000000000000000 P f7480f49edbd4746ac36c92674ce0524 [sys.catalog]: This master's current role is: LEADER
21:22:16.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:16.095533 10337 debug-util.cc:398] Leaking SignalData structure 0x7b0800089fe0 after lost signal to thread 10274
21:22:16.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:16.096246 10337 debug-util.cc:398] Leaking SignalData structure 0x7b0800095780 after lost signal to thread 10340
21:22:16.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:16.098127 11295 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
21:22:16.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:16.100581 11082 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.186.60" port: 38107 } attempt: 7
21:22:16.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:16.114214 11295 catalog_manager.cc:1494] Initializing Kudu cluster ID...
21:22:16.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 28 21:22:16 dist-test-slave-hq3s krb5kdc[10996](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1769635331, etypes {rep=17 tkt=17 ses=17}, kudu/127.10.186.62@KRBTEST.COM for kudu/127.10.186.60@KRBTEST.COM
21:22:16.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:16.158120 11142 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 5ada6348d26f452da1cd3c3f9f9294f4 [term 1 FOLLOWER]: Refusing update from remote peer f7480f49edbd4746ac36c92674ce0524: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
21:22:16.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:16.161265 11265 consensus_queue.cc:1048] T 00000000000000000000000000000000 P f7480f49edbd4746ac36c92674ce0524 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5ada6348d26f452da1cd3c3f9f9294f4" member_type: VOTER last_known_addr { host: "127.10.186.61" port: 45819 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
21:22:16.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:16.199175 11082 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 04f29f2996a546f79797e6e6920a700f: Bootstrap starting.
21:22:16.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:16.210906 11082 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 04f29f2996a546f79797e6e6920a700f: Neither blocks nor log segments found. Creating new log.
21:22:16.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:16.222469 11280 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5ada6348d26f452da1cd3c3f9f9294f4 [sys.catalog]: SysCatalogTable state changed. Reason: New leader f7480f49edbd4746ac36c92674ce0524. Latest consensus state: current_term: 1 leader_uuid: "f7480f49edbd4746ac36c92674ce0524" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "04f29f2996a546f79797e6e6920a700f" member_type: VOTER last_known_addr { host: "127.10.186.62" port: 46125 } } peers { permanent_uuid: "5ada6348d26f452da1cd3c3f9f9294f4" member_type: VOTER last_known_addr { host: "127.10.186.61" port: 45819 } } peers { permanent_uuid: "f7480f49edbd4746ac36c92674ce0524" member_type: VOTER last_known_addr { host: "127.10.186.60" port: 38107 } } }
21:22:16.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:16.223196 11280 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5ada6348d26f452da1cd3c3f9f9294f4 [sys.catalog]: This master's current role is: FOLLOWER
21:22:16.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:16.224598 11082 log.cc:826] T 00000000000000000000000000000000 P 04f29f2996a546f79797e6e6920a700f: Log is configured to *not* fsync() on all Append() calls
21:22:16.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:16.230942 11298 mvcc.cc:204] Tried to move back new op lower bound from 7248426336880517120 to 7248426336574648320. Current Snapshot: MvccSnapshot[applied={T|T < 7248426336880517120}]
21:22:16.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:16.238425 11280 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5ada6348d26f452da1cd3c3f9f9294f4 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "f7480f49edbd4746ac36c92674ce0524" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "04f29f2996a546f79797e6e6920a700f" member_type: VOTER last_known_addr { host: "127.10.186.62" port: 46125 } } peers { permanent_uuid: "5ada6348d26f452da1cd3c3f9f9294f4" member_type: VOTER last_known_addr { host: "127.10.186.61" port: 45819 } } peers { permanent_uuid: "f7480f49edbd4746ac36c92674ce0524" member_type: VOTER last_known_addr { host: "127.10.186.60" port: 38107 } } }
21:22:16.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:16.239197 11280 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5ada6348d26f452da1cd3c3f9f9294f4 [sys.catalog]: This master's current role is: FOLLOWER
21:22:16.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:16.250851 11265 sys_catalog.cc:455] T 00000000000000000000000000000000 P f7480f49edbd4746ac36c92674ce0524 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "f7480f49edbd4746ac36c92674ce0524" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "04f29f2996a546f79797e6e6920a700f" member_type: VOTER last_known_addr { host: "127.10.186.62" port: 46125 } } peers { permanent_uuid: "5ada6348d26f452da1cd3c3f9f9294f4" member_type: VOTER last_known_addr { host: "127.10.186.61" port: 45819 } } peers { permanent_uuid: "f7480f49edbd4746ac36c92674ce0524" member_type: VOTER last_known_addr { host: "127.10.186.60" port: 38107 } } }
21:22:16.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:16.251657 11265 sys_catalog.cc:458] T 00000000000000000000000000000000 P f7480f49edbd4746ac36c92674ce0524 [sys.catalog]: This master's current role is: LEADER
21:22:16.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:16.252835 11295 catalog_manager.cc:1357] Generated new cluster ID: 3e7dc32b40b94de0a90f622e8e1e29df
21:22:16.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:16.253139 11295 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
21:22:16.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:16.259382 11082 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 04f29f2996a546f79797e6e6920a700f: No bootstrap required, opened a new log
21:22:16.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:16.298069 11295 catalog_manager.cc:1380] Generated new certificate authority record
21:22:16.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:16.300161 11295 catalog_manager.cc:1514] Loading token signing keys...
21:22:16.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:16.311048 11082 raft_consensus.cc:359] T 00000000000000000000000000000000 P 04f29f2996a546f79797e6e6920a700f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "04f29f2996a546f79797e6e6920a700f" member_type: VOTER last_known_addr { host: "127.10.186.62" port: 46125 } } peers { permanent_uuid: "5ada6348d26f452da1cd3c3f9f9294f4" member_type: VOTER last_known_addr { host: "127.10.186.61" port: 45819 } } peers { permanent_uuid: "f7480f49edbd4746ac36c92674ce0524" member_type: VOTER last_known_addr { host: "127.10.186.60" port: 38107 } }
21:22:16.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:16.329216 11082 raft_consensus.cc:385] T 00000000000000000000000000000000 P 04f29f2996a546f79797e6e6920a700f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:22:16.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:16.329785 11082 raft_consensus.cc:740] T 00000000000000000000000000000000 P 04f29f2996a546f79797e6e6920a700f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 04f29f2996a546f79797e6e6920a700f, State: Initialized, Role: FOLLOWER
21:22:16.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:16.332412 11082 consensus_queue.cc:260] T 00000000000000000000000000000000 P 04f29f2996a546f79797e6e6920a700f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "04f29f2996a546f79797e6e6920a700f" member_type: VOTER last_known_addr { host: "127.10.186.62" port: 46125 } } peers { permanent_uuid: "5ada6348d26f452da1cd3c3f9f9294f4" member_type: VOTER last_known_addr { host: "127.10.186.61" port: 45819 } } peers { permanent_uuid: "f7480f49edbd4746ac36c92674ce0524" member_type: VOTER last_known_addr { host: "127.10.186.60" port: 38107 } }
21:22:16.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:16.334717 11295 catalog_manager.cc:6027] T 00000000000000000000000000000000 P f7480f49edbd4746ac36c92674ce0524: Generated new TSK 0
21:22:16.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:16.335851 11295 catalog_manager.cc:1524] Initializing in-progress tserver states...
21:22:16.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:16.346668 11082 sys_catalog.cc:565] T 00000000000000000000000000000000 P 04f29f2996a546f79797e6e6920a700f [sys.catalog]: configured and running, proceeding with master startup.
21:22:16.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:16.347164 11055 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 04f29f2996a546f79797e6e6920a700f [term 0 FOLLOWER]: Advancing to term 1
21:22:16.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:16.353065 11056 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "f7480f49edbd4746ac36c92674ce0524" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "04f29f2996a546f79797e6e6920a700f"
21:22:16.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:16.355859 11055 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 04f29f2996a546f79797e6e6920a700f [term 1 FOLLOWER]: Refusing update from remote peer f7480f49edbd4746ac36c92674ce0524: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
21:22:16.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:16.357432 11293 consensus_queue.cc:1048] T 00000000000000000000000000000000 P f7480f49edbd4746ac36c92674ce0524 [LEADER]: Connected to new peer: Peer: permanent_uuid: "04f29f2996a546f79797e6e6920a700f" member_type: VOTER last_known_addr { host: "127.10.186.62" port: 46125 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
21:22:16.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:16.358889 11057 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "f7480f49edbd4746ac36c92674ce0524" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "04f29f2996a546f79797e6e6920a700f" is_pre_election: true
21:22:16.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:16.359213 11303 sys_catalog.cc:455] T 00000000000000000000000000000000 P 04f29f2996a546f79797e6e6920a700f [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "04f29f2996a546f79797e6e6920a700f" member_type: VOTER last_known_addr { host: "127.10.186.62" port: 46125 } } peers { permanent_uuid: "5ada6348d26f452da1cd3c3f9f9294f4" member_type: VOTER last_known_addr { host: "127.10.186.61" port: 45819 } } peers { permanent_uuid: "f7480f49edbd4746ac36c92674ce0524" member_type: VOTER last_known_addr { host: "127.10.186.60" port: 38107 } } }
21:22:16.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:16.359719 11057 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 04f29f2996a546f79797e6e6920a700f [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate f7480f49edbd4746ac36c92674ce0524 in term 1.
21:22:16.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:16.359970 11303 sys_catalog.cc:458] T 00000000000000000000000000000000 P 04f29f2996a546f79797e6e6920a700f [sys.catalog]: This master's current role is: FOLLOWER
21:22:16.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:16.420773 11309 sys_catalog.cc:455] T 00000000000000000000000000000000 P 04f29f2996a546f79797e6e6920a700f [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "f7480f49edbd4746ac36c92674ce0524" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "04f29f2996a546f79797e6e6920a700f" member_type: VOTER last_known_addr { host: "127.10.186.62" port: 46125 } } peers { permanent_uuid: "5ada6348d26f452da1cd3c3f9f9294f4" member_type: VOTER last_known_addr { host: "127.10.186.61" port: 45819 } } peers { permanent_uuid: "f7480f49edbd4746ac36c92674ce0524" member_type: VOTER last_known_addr { host: "127.10.186.60" port: 38107 } } }
21:22:16.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:16.421593 11309 sys_catalog.cc:458] T 00000000000000000000000000000000 P 04f29f2996a546f79797e6e6920a700f [sys.catalog]: This master's current role is: FOLLOWER
21:22:16.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:16.422516 11303 sys_catalog.cc:455] T 00000000000000000000000000000000 P 04f29f2996a546f79797e6e6920a700f [sys.catalog]: SysCatalogTable state changed. Reason: New leader f7480f49edbd4746ac36c92674ce0524. Latest consensus state: current_term: 1 leader_uuid: "f7480f49edbd4746ac36c92674ce0524" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "04f29f2996a546f79797e6e6920a700f" member_type: VOTER last_known_addr { host: "127.10.186.62" port: 46125 } } peers { permanent_uuid: "5ada6348d26f452da1cd3c3f9f9294f4" member_type: VOTER last_known_addr { host: "127.10.186.61" port: 45819 } } peers { permanent_uuid: "f7480f49edbd4746ac36c92674ce0524" member_type: VOTER last_known_addr { host: "127.10.186.60" port: 38107 } } }
21:22:16.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:16.423149 11303 sys_catalog.cc:458] T 00000000000000000000000000000000 P 04f29f2996a546f79797e6e6920a700f [sys.catalog]: This master's current role is: FOLLOWER
21:22:16.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:16.504997 11319 catalog_manager.cc:1269] Loaded cluster ID: 3e7dc32b40b94de0a90f622e8e1e29df
21:22:16.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:16.505496 11319 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 04f29f2996a546f79797e6e6920a700f: loading cluster ID for follower catalog manager: success
21:22:16.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:16.513231 11319 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 04f29f2996a546f79797e6e6920a700f: acquiring CA information for follower catalog manager: success
21:22:16.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:16.524634 11319 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 04f29f2996a546f79797e6e6920a700f: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
21:22:16.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:16.836004 11291 catalog_manager.cc:1269] Loaded cluster ID: 3e7dc32b40b94de0a90f622e8e1e29df
21:22:16.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:16.836527 11291 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 5ada6348d26f452da1cd3c3f9f9294f4: loading cluster ID for follower catalog manager: success
21:22:16.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:16.852289 11291 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 5ada6348d26f452da1cd3c3f9f9294f4: acquiring CA information for follower catalog manager: success
21:22:16.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:16.858369 11291 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 5ada6348d26f452da1cd3c3f9f9294f4: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
21:22:16.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:16.891942 11321 sys_catalog.cc:455] T 00000000000000000000000000000000 P f7480f49edbd4746ac36c92674ce0524 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "f7480f49edbd4746ac36c92674ce0524" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "04f29f2996a546f79797e6e6920a700f" member_type: VOTER last_known_addr { host: "127.10.186.62" port: 46125 } } peers { permanent_uuid: "5ada6348d26f452da1cd3c3f9f9294f4" member_type: VOTER last_known_addr { host: "127.10.186.61" port: 45819 } } peers { permanent_uuid: "f7480f49edbd4746ac36c92674ce0524" member_type: VOTER last_known_addr { host: "127.10.186.60" port: 38107 } } }
21:22:16.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:16.892709 11321 sys_catalog.cc:458] T 00000000000000000000000000000000 P f7480f49edbd4746ac36c92674ce0524 [sys.catalog]: This master's current role is: LEADER
21:22:18.630 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.10.186.60:38107
21:22:18.630 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
21:22:18.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:18.631482 10984 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskp7EThw/build/tsan/bin/kudu with pid 11177
21:22:18.693 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:22:18.693 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.10.186.61:45819
21:22:18.693 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
21:22:18.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:18.694315 10984 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskp7EThw/build/tsan/bin/kudu with pid 11091
21:22:18.733 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:22:18.734 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.10.186.62:46125
21:22:18.734 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
21:22:18.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:18.734489 10984 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskp7EThw/build/tsan/bin/kudu with pid 11012
21:22:18.760 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:22:18.760 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.10.186.60:38107
21:22:18.760 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
21:22:18.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:18.762140 10984 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskp7EThw/build/tsan/bin/kudu
21:22:18.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskp7EThw/build/tsan/bin/kudu
21:22:18.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/master-2/wal
21:22:18.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/master-2/data
21:22:18.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:22:18.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:22:18.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:22:18.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:22:18.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:22:18.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:22:18.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/master-2/logs
21:22:18.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/master-2/data/info.pb
21:22:18.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:22:18.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:22:18.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:22:18.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:22:18.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:22:18.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:22:18.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:22:18.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:22:18.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:22:18.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:22:18.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:22:18.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:22:18.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.186.60:38107
21:22:18.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.186.60
21:22:18.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36489
21:22:18.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/krb5kdc/kudu.keytab
21:22:18.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.10.186.60
21:22:18.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:22:18.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
21:22:18.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
21:22:18.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
21:22:18.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.186.20:34545
21:22:18.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:22:18.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:22:18.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:22:18.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:22:18.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.186.62:46125,127.10.186.61:45819,127.10.186.60:38107
21:22:18.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:22:18.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34709/jwks.json
21:22:18.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
21:22:18.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
21:22:19.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:19.059675 11326 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:22:19.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:19.060200 11326 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:22:19.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:19.060823 11326 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:22:19.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:19.094514 11326 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
21:22:19.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:19.094756 11326 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:22:19.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:19.095021 11326 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:22:19.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:19.095218 11326 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:22:19.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:19.095410 11326 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:22:19.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:19.095654 11326 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:22:19.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:19.095818 11326 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34709/jwks.json
21:22:19.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:19.133693 11326 master_runner.cc:386] Master server non-default flags:
21:22:19.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:22:19.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.186.20:34545
21:22:19.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:22:19.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:22:19.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/master-2/data
21:22:19.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/master-2/wal
21:22:19.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
21:22:19.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:22:19.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.186.62:46125,127.10.186.61:45819,127.10.186.60:38107
21:22:19.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/krb5kdc/kudu.keytab
21:22:19.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.10.186.60
21:22:19.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:22:19.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:22:19.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:22:19.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.186.60:38107
21:22:19.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:22:19.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:22:19.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:22:19.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34709/jwks.json
21:22:19.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:22:19.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
21:22:19.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
21:22:19.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:22:19.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:22:19.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/master-2/data/info.pb
21:22:19.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.186.60
21:22:19.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36489
21:22:19.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:22:19.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:22:19.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:22:19.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:22:19.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:22:19.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/master-2/logs
21:22:19.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:22:19.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:22:19.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:22:19.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:22:19.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:22:19.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3722ed49c19ce54c7feae2fa944e32b1d4ed8d2d
21:22:19.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:22:19.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 28 Jan 2026 21:02:00 UTC on 24a791456cd2
21:22:19.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10237
21:22:19.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:22:19.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:19.134948 11326 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:22:19.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:19.135957 11326 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:22:19.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:19.146605 11332 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:22:20.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:19.147845 11333 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:22:20.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:19.152172 11335 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:22:20.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:20.312958 11334 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1161 milliseconds
21:22:20.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:20.313071 11326 server_base.cc:1055] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:22:20.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:20.314272 11326 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:22:20.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:20.316821 11326 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:22:20.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:20.318153 11326 hybrid_clock.cc:648] HybridClock initialized: now 1769635340318126 us; error 40 us; skew 500 ppm
21:22:20.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:20.318919 11326 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:22:20.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 28 21:22:20 dist-test-slave-hq3s krb5kdc[10996](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1769635340, etypes {rep=17 tkt=17 ses=17}, kudu/127.10.186.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:22:20.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:20.324673 11326 init.cc:377] Logged in from keytab as kudu/127.10.186.60@KRBTEST.COM (short username kudu)
21:22:20.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:20.329564 11326 webserver.cc:492] Webserver started at http://127.10.186.60:36489/ using document root <none> and password file <none>
21:22:20.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:20.330425 11326 fs_manager.cc:362] Metadata directory not provided
21:22:20.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:20.330621 11326 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:22:20.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:20.338299 11326 fs_manager.cc:714] Time spent opening directory manager: real 0.005s user 0.005s sys 0.000s
21:22:20.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:20.342746 11344 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:22:20.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:20.343894 11326 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
21:22:20.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:20.344250 11326 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/master-2/data,/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/master-2/wal
21:22:20.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f7480f49edbd4746ac36c92674ce0524"
21:22:20.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-01-28 21:22:15 on dist-test-slave-hq3s"
21:22:20.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:20.346218 11326 fs_report.cc:389] FS layout report
21:22:20.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:22:20.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/master-2/wal
21:22:20.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/master-2/wal
21:22:20.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/master-2/data/data
21:22:20.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:22:20.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:22:20.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:22:20.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:22:20.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:22:20.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:22:20.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:22:20.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:22:20.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:22:20.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:22:20.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:22:20.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:20.393823 11326 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:22:20.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:20.402933 11326 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:22:20.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:20.403424 11326 kserver.cc:163] Server-wide thread pool size limit: 3276
21:22:20.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:20.472376 11326 rpc_server.cc:307] RPC server started. Bound to: 127.10.186.60:38107
21:22:20.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:20.472484 11395 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.10.186.60:38107 every 8 connection(s)
21:22:20.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:20.475296 11326 server_base.cc:1192] Dumped server information to /tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/master-2/data/info.pb
21:22:20.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:20.478896 10984 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskp7EThw/build/tsan/bin/kudu as pid 11326
21:22:20.479 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:22:20.479 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.10.186.61:45819
21:22:20.479 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
21:22:20.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:20.480829 10984 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskp7EThw/build/tsan/bin/kudu
21:22:20.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskp7EThw/build/tsan/bin/kudu
21:22:20.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/master-1/wal
21:22:20.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/master-1/data
21:22:20.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:22:20.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:22:20.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:22:20.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:22:20.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:22:20.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:22:20.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/master-1/logs
21:22:20.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/master-1/data/info.pb
21:22:20.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:22:20.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:22:20.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:22:20.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:22:20.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:22:20.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:22:20.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:22:20.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:22:20.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:22:20.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:22:20.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:22:20.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:22:20.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.186.61:45819
21:22:20.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.186.61
21:22:20.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33271
21:22:20.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/krb5kdc/kudu.keytab
21:22:20.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.10.186.61
21:22:20.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:22:20.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
21:22:20.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
21:22:20.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
21:22:20.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.186.20:34545
21:22:20.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:22:20.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:22:20.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:22:20.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:22:20.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.186.62:46125,127.10.186.61:45819,127.10.186.60:38107
21:22:20.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:22:20.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34709/jwks.json
21:22:20.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
21:22:20.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
21:22:20.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:20.486768 11396 sys_catalog.cc:263] Verifying existing consensus state
21:22:20.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:20.492092 11396 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f7480f49edbd4746ac36c92674ce0524: Bootstrap starting.
21:22:20.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:20.530306 11396 log.cc:826] T 00000000000000000000000000000000 P f7480f49edbd4746ac36c92674ce0524: Log is configured to *not* fsync() on all Append() calls
21:22:20.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:20.546047 11396 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f7480f49edbd4746ac36c92674ce0524: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
21:22:20.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:20.546998 11396 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f7480f49edbd4746ac36c92674ce0524: Bootstrap complete.
21:22:20.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:20.569053 11396 raft_consensus.cc:359] T 00000000000000000000000000000000 P f7480f49edbd4746ac36c92674ce0524 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "04f29f2996a546f79797e6e6920a700f" member_type: VOTER last_known_addr { host: "127.10.186.62" port: 46125 } } peers { permanent_uuid: "5ada6348d26f452da1cd3c3f9f9294f4" member_type: VOTER last_known_addr { host: "127.10.186.61" port: 45819 } } peers { permanent_uuid: "f7480f49edbd4746ac36c92674ce0524" member_type: VOTER last_known_addr { host: "127.10.186.60" port: 38107 } }
21:22:20.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:20.571239 11396 raft_consensus.cc:740] T 00000000000000000000000000000000 P f7480f49edbd4746ac36c92674ce0524 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: f7480f49edbd4746ac36c92674ce0524, State: Initialized, Role: FOLLOWER
21:22:20.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:20.572172 11396 consensus_queue.cc:260] T 00000000000000000000000000000000 P f7480f49edbd4746ac36c92674ce0524 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "04f29f2996a546f79797e6e6920a700f" member_type: VOTER last_known_addr { host: "127.10.186.62" port: 46125 } } peers { permanent_uuid: "5ada6348d26f452da1cd3c3f9f9294f4" member_type: VOTER last_known_addr { host: "127.10.186.61" port: 45819 } } peers { permanent_uuid: "f7480f49edbd4746ac36c92674ce0524" member_type: VOTER last_known_addr { host: "127.10.186.60" port: 38107 } }
21:22:20.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:20.575026 11400 sys_catalog.cc:455] T 00000000000000000000000000000000 P f7480f49edbd4746ac36c92674ce0524 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "04f29f2996a546f79797e6e6920a700f" member_type: VOTER last_known_addr { host: "127.10.186.62" port: 46125 } } peers { permanent_uuid: "5ada6348d26f452da1cd3c3f9f9294f4" member_type: VOTER last_known_addr { host: "127.10.186.61" port: 45819 } } peers { permanent_uuid: "f7480f49edbd4746ac36c92674ce0524" member_type: VOTER last_known_addr { host: "127.10.186.60" port: 38107 } } }
21:22:20.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:20.575832 11400 sys_catalog.cc:458] T 00000000000000000000000000000000 P f7480f49edbd4746ac36c92674ce0524 [sys.catalog]: This master's current role is: FOLLOWER
21:22:20.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:20.576786 11396 sys_catalog.cc:565] T 00000000000000000000000000000000 P f7480f49edbd4746ac36c92674ce0524 [sys.catalog]: configured and running, proceeding with master startup.
21:22:20.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:20.605840 11411 catalog_manager.cc:1269] Loaded cluster ID: 3e7dc32b40b94de0a90f622e8e1e29df
21:22:20.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:20.606046 11411 catalog_manager.cc:1562] T 00000000000000000000000000000000 P f7480f49edbd4746ac36c92674ce0524: loading cluster ID for follower catalog manager: success
21:22:20.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:20.612967 11411 catalog_manager.cc:1584] T 00000000000000000000000000000000 P f7480f49edbd4746ac36c92674ce0524: acquiring CA information for follower catalog manager: success
21:22:20.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:20.618206 11411 catalog_manager.cc:1612] T 00000000000000000000000000000000 P f7480f49edbd4746ac36c92674ce0524: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
21:22:20.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:20.607553 11326 master_runner.cc:186] Error getting master registration for 127.10.186.62:46125: Network error: Client connection negotiation failed: client connection to 127.10.186.62:46125: connect: Connection refused (error 111),
21:22:20.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:20.628283 11326 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
21:22:20.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:20.822698 11398 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:22:20.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:20.823302 11398 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:22:20.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:20.824057 11398 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:22:20.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:20.857159 11398 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
21:22:20.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:20.857422 11398 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:22:20.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:20.857811 11398 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:22:20.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:20.858088 11398 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:22:20.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:20.858350 11398 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:22:20.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:20.858615 11398 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:22:20.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:20.858827 11398 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34709/jwks.json
21:22:20.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:20.896896 11398 master_runner.cc:386] Master server non-default flags:
21:22:20.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:22:20.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.186.20:34545
21:22:20.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:22:20.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:22:20.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/master-1/data
21:22:20.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/master-1/wal
21:22:20.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
21:22:20.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:22:20.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.186.62:46125,127.10.186.61:45819,127.10.186.60:38107
21:22:20.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/krb5kdc/kudu.keytab
21:22:20.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.10.186.61
21:22:20.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:22:20.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:22:20.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:22:20.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.186.61:45819
21:22:20.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:22:20.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:22:20.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:22:20.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34709/jwks.json
21:22:20.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:22:20.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
21:22:20.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
21:22:20.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:22:20.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:22:20.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/master-1/data/info.pb
21:22:20.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.186.61
21:22:20.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33271
21:22:20.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:22:20.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:22:20.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:22:20.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:22:20.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:22:20.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/master-1/logs
21:22:20.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:22:20.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:22:20.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:22:20.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:22:20.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:22:20.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3722ed49c19ce54c7feae2fa944e32b1d4ed8d2d
21:22:20.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:22:20.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 28 Jan 2026 21:02:00 UTC on 24a791456cd2
21:22:20.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10237
21:22:20.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:22:20.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:20.898452 11398 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:22:20.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:20.899670 11398 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:22:20.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:20.912596 11420 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:22:21.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:21.948415 11425 raft_consensus.cc:493] T 00000000000000000000000000000000 P f7480f49edbd4746ac36c92674ce0524 [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:22:21.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:21.949090 11425 raft_consensus.cc:515] T 00000000000000000000000000000000 P f7480f49edbd4746ac36c92674ce0524 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "04f29f2996a546f79797e6e6920a700f" member_type: VOTER last_known_addr { host: "127.10.186.62" port: 46125 } } peers { permanent_uuid: "5ada6348d26f452da1cd3c3f9f9294f4" member_type: VOTER last_known_addr { host: "127.10.186.61" port: 45819 } } peers { permanent_uuid: "f7480f49edbd4746ac36c92674ce0524" member_type: VOTER last_known_addr { host: "127.10.186.60" port: 38107 } }
21:22:21.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:21.963054 11347 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.10.186.62:46125: connect: Connection refused (error 111)
21:22:21.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:21.966316 11425 leader_election.cc:290] T 00000000000000000000000000000000 P f7480f49edbd4746ac36c92674ce0524 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 04f29f2996a546f79797e6e6920a700f (127.10.186.62:46125), 5ada6348d26f452da1cd3c3f9f9294f4 (127.10.186.61:45819)
21:22:22.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:22.003492 11347 leader_election.cc:336] T 00000000000000000000000000000000 P f7480f49edbd4746ac36c92674ce0524 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 04f29f2996a546f79797e6e6920a700f (127.10.186.62:46125): Network error: Client connection negotiation failed: client connection to 127.10.186.62:46125: connect: Connection refused (error 111)
21:22:22.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:22.004351 11347 leader_election.cc:336] T 00000000000000000000000000000000 P f7480f49edbd4746ac36c92674ce0524 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 5ada6348d26f452da1cd3c3f9f9294f4 (127.10.186.61:45819): Network error: Client connection negotiation failed: client connection to 127.10.186.61:45819: connect: Connection refused (error 111)
21:22:22.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:22.004889 11347 leader_election.cc:304] T 00000000000000000000000000000000 P f7480f49edbd4746ac36c92674ce0524 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: f7480f49edbd4746ac36c92674ce0524; no voters: 04f29f2996a546f79797e6e6920a700f, 5ada6348d26f452da1cd3c3f9f9294f4
21:22:22.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:22.006103 11425 raft_consensus.cc:2749] T 00000000000000000000000000000000 P f7480f49edbd4746ac36c92674ce0524 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
21:22:22.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:22.312992 11418 debug-util.cc:398] Leaking SignalData structure 0x7b0800034ca0 after lost signal to thread 11398
21:22:22.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:22.789201 11398 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.878s user 0.663s sys 0.975s
21:22:22.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:22.789991 11421 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1876 milliseconds
21:22:22.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:22.790388 11398 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.879s user 0.663s sys 0.975s
21:22:22.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:20.913336 11419 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:22:22.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:22.793223 11422 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:22:22.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:22.793187 11398 server_base.cc:1055] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:22:22.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:22.795156 11398 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:22:22.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:22.797570 11398 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:22:22.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:22.798929 11398 hybrid_clock.cc:648] HybridClock initialized: now 1769635342798864 us; error 82 us; skew 500 ppm
21:22:22.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:22.799787 11398 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:22:22.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 28 21:22:22 dist-test-slave-hq3s krb5kdc[10996](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1769635342, etypes {rep=17 tkt=17 ses=17}, kudu/127.10.186.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:22:22.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:22.805785 11398 init.cc:377] Logged in from keytab as kudu/127.10.186.61@KRBTEST.COM (short username kudu)
21:22:22.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:22.810887 11398 webserver.cc:492] Webserver started at http://127.10.186.61:33271/ using document root <none> and password file <none>
21:22:22.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:22.811936 11398 fs_manager.cc:362] Metadata directory not provided
21:22:22.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:22.812175 11398 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:22:22.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:22.822237 11398 fs_manager.cc:714] Time spent opening directory manager: real 0.007s user 0.006s sys 0.003s
21:22:22.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:22.827020 11434 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:22:22.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:22.828284 11398 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
21:22:22.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:22.828637 11398 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/master-1/data,/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/master-1/wal
21:22:22.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5ada6348d26f452da1cd3c3f9f9294f4"
21:22:22.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-01-28 21:22:13 on dist-test-slave-hq3s"
21:22:22.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:22.830536 11398 fs_report.cc:389] FS layout report
21:22:22.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:22:22.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/master-1/wal
21:22:22.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/master-1/wal
21:22:22.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/master-1/data/data
21:22:22.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:22:22.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:22:22.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:22:22.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:22:22.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:22:22.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:22:22.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:22:22.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:22:22.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:22:22.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:22:22.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:22:22.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:22.884359 11398 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:22:22.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:22.889739 11398 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:22:22.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:22.890199 11398 kserver.cc:163] Server-wide thread pool size limit: 3276
21:22:22.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:22.983086 11398 rpc_server.cc:307] RPC server started. Bound to: 127.10.186.61:45819
21:22:22.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:22.983143 11485 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.10.186.61:45819 every 8 connection(s)
21:22:22.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:22.986310 11398 server_base.cc:1192] Dumped server information to /tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/master-1/data/info.pb
21:22:22.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:22.996773 10984 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskp7EThw/build/tsan/bin/kudu as pid 11398
21:22:22.997 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:22:22.997 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.10.186.62:46125
21:22:22.997 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
21:22:22.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:22.999259 10984 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskp7EThw/build/tsan/bin/kudu
21:22:22.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskp7EThw/build/tsan/bin/kudu
21:22:22.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/master-0/wal
21:22:23.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/master-0/data
21:22:23.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:22:23.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:22:23.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:22:23.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:22:23.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:22:23.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:22:23.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/master-0/logs
21:22:23.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/master-0/data/info.pb
21:22:23.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:22:23.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:22:23.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:22:23.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:22:23.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:22:23.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:22:23.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:22:23.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:22:23.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:22:23.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:22:23.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:22:23.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:22:23.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.186.62:46125
21:22:23.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.186.62
21:22:23.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37867
21:22:23.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/krb5kdc/kudu.keytab
21:22:23.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.10.186.62
21:22:23.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:22:23.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
21:22:23.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
21:22:23.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
21:22:23.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.186.20:34545
21:22:23.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:22:23.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:22:23.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:22:23.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:22:23.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.186.62:46125,127.10.186.61:45819,127.10.186.60:38107
21:22:23.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:22:23.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34709/jwks.json
21:22:23.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
21:22:23.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
21:22:23.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:23.002925 11486 sys_catalog.cc:263] Verifying existing consensus state
21:22:23.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:23.010552 11486 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5ada6348d26f452da1cd3c3f9f9294f4: Bootstrap starting.
21:22:23.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:23.065181 11486 log.cc:826] T 00000000000000000000000000000000 P 5ada6348d26f452da1cd3c3f9f9294f4: Log is configured to *not* fsync() on all Append() calls
21:22:23.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:23.085000 11486 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5ada6348d26f452da1cd3c3f9f9294f4: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
21:22:23.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:23.085978 11486 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5ada6348d26f452da1cd3c3f9f9294f4: Bootstrap complete.
21:22:23.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:23.119242 11486 raft_consensus.cc:359] T 00000000000000000000000000000000 P 5ada6348d26f452da1cd3c3f9f9294f4 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "04f29f2996a546f79797e6e6920a700f" member_type: VOTER last_known_addr { host: "127.10.186.62" port: 46125 } } peers { permanent_uuid: "5ada6348d26f452da1cd3c3f9f9294f4" member_type: VOTER last_known_addr { host: "127.10.186.61" port: 45819 } } peers { permanent_uuid: "f7480f49edbd4746ac36c92674ce0524" member_type: VOTER last_known_addr { host: "127.10.186.60" port: 38107 } }
21:22:23.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:23.122539 11486 raft_consensus.cc:740] T 00000000000000000000000000000000 P 5ada6348d26f452da1cd3c3f9f9294f4 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5ada6348d26f452da1cd3c3f9f9294f4, State: Initialized, Role: FOLLOWER
21:22:23.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:23.123561 11486 consensus_queue.cc:260] T 00000000000000000000000000000000 P 5ada6348d26f452da1cd3c3f9f9294f4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "04f29f2996a546f79797e6e6920a700f" member_type: VOTER last_known_addr { host: "127.10.186.62" port: 46125 } } peers { permanent_uuid: "5ada6348d26f452da1cd3c3f9f9294f4" member_type: VOTER last_known_addr { host: "127.10.186.61" port: 45819 } } peers { permanent_uuid: "f7480f49edbd4746ac36c92674ce0524" member_type: VOTER last_known_addr { host: "127.10.186.60" port: 38107 } }
21:22:23.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:23.126484 11490 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5ada6348d26f452da1cd3c3f9f9294f4 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "04f29f2996a546f79797e6e6920a700f" member_type: VOTER last_known_addr { host: "127.10.186.62" port: 46125 } } peers { permanent_uuid: "5ada6348d26f452da1cd3c3f9f9294f4" member_type: VOTER last_known_addr { host: "127.10.186.61" port: 45819 } } peers { permanent_uuid: "f7480f49edbd4746ac36c92674ce0524" member_type: VOTER last_known_addr { host: "127.10.186.60" port: 38107 } } }
21:22:23.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:23.128016 11490 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5ada6348d26f452da1cd3c3f9f9294f4 [sys.catalog]: This master's current role is: FOLLOWER
21:22:23.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:23.128391 11486 sys_catalog.cc:565] T 00000000000000000000000000000000 P 5ada6348d26f452da1cd3c3f9f9294f4 [sys.catalog]: configured and running, proceeding with master startup.
21:22:23.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:23.211946 11502 catalog_manager.cc:1269] Loaded cluster ID: 3e7dc32b40b94de0a90f622e8e1e29df
21:22:23.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:23.212589 11502 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 5ada6348d26f452da1cd3c3f9f9294f4: loading cluster ID for follower catalog manager: success
21:22:23.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:23.220477 11502 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 5ada6348d26f452da1cd3c3f9f9294f4: acquiring CA information for follower catalog manager: success
21:22:23.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:23.226981 11502 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 5ada6348d26f452da1cd3c3f9f9294f4: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
21:22:23.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:23.201140 11398 master_runner.cc:186] Error getting master registration for 127.10.186.62:46125: Network error: Client connection negotiation failed: client connection to 127.10.186.62:46125: connect: Connection refused (error 111),
21:22:23.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:23.228199 11398 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
21:22:23.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:23.417644 11488 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:22:23.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:23.418447 11488 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:22:23.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:23.419562 11488 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:22:23.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:23.476572 11488 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
21:22:23.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:23.476903 11488 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:22:23.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:23.477352 11488 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:22:23.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:23.477677 11488 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:22:23.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:23.478004 11488 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:22:23.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:23.478317 11488 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:22:23.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:23.478547 11488 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34709/jwks.json
21:22:23.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:23.544358 11488 master_runner.cc:386] Master server non-default flags:
21:22:23.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:22:23.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.186.20:34545
21:22:23.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:22:23.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:22:23.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/master-0/data
21:22:23.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/master-0/wal
21:22:23.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
21:22:23.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:22:23.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.186.62:46125,127.10.186.61:45819,127.10.186.60:38107
21:22:23.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/krb5kdc/kudu.keytab
21:22:23.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.10.186.62
21:22:23.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:22:23.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:22:23.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:22:23.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.186.62:46125
21:22:23.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:22:23.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:22:23.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:22:23.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34709/jwks.json
21:22:23.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:22:23.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
21:22:23.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
21:22:23.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:22:23.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:22:23.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/master-0/data/info.pb
21:22:23.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.186.62
21:22:23.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37867
21:22:23.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:22:23.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:22:23.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:22:23.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:22:23.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:22:23.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/master-0/logs
21:22:23.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:22:23.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:22:23.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:22:23.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:22:23.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:22:23.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3722ed49c19ce54c7feae2fa944e32b1d4ed8d2d
21:22:23.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:22:23.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 28 Jan 2026 21:02:00 UTC on 24a791456cd2
21:22:23.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10237
21:22:23.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:22:23.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:23.546029 11488 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:22:23.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:23.547365 11488 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:22:23.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:23.560190 11511 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:22:24.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:24.048775 11517 raft_consensus.cc:493] T 00000000000000000000000000000000 P f7480f49edbd4746ac36c92674ce0524 [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:22:24.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:24.049257 11517 raft_consensus.cc:515] T 00000000000000000000000000000000 P f7480f49edbd4746ac36c92674ce0524 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "04f29f2996a546f79797e6e6920a700f" member_type: VOTER last_known_addr { host: "127.10.186.62" port: 46125 } } peers { permanent_uuid: "5ada6348d26f452da1cd3c3f9f9294f4" member_type: VOTER last_known_addr { host: "127.10.186.61" port: 45819 } } peers { permanent_uuid: "f7480f49edbd4746ac36c92674ce0524" member_type: VOTER last_known_addr { host: "127.10.186.60" port: 38107 } }
21:22:24.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:24.051110 11517 leader_election.cc:290] T 00000000000000000000000000000000 P f7480f49edbd4746ac36c92674ce0524 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 04f29f2996a546f79797e6e6920a700f (127.10.186.62:46125), 5ada6348d26f452da1cd3c3f9f9294f4 (127.10.186.61:45819)
21:22:24.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:24.073688 11347 leader_election.cc:336] T 00000000000000000000000000000000 P f7480f49edbd4746ac36c92674ce0524 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 04f29f2996a546f79797e6e6920a700f (127.10.186.62:46125): Network error: Client connection negotiation failed: client connection to 127.10.186.62:46125: connect: Connection refused (error 111)
21:22:24.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:24.077644 11461 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "f7480f49edbd4746ac36c92674ce0524" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "5ada6348d26f452da1cd3c3f9f9294f4" is_pre_election: true
21:22:24.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:24.078730 11461 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 5ada6348d26f452da1cd3c3f9f9294f4 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate f7480f49edbd4746ac36c92674ce0524 in term 1.
21:22:24.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:24.080801 11347 leader_election.cc:304] T 00000000000000000000000000000000 P f7480f49edbd4746ac36c92674ce0524 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 5ada6348d26f452da1cd3c3f9f9294f4, f7480f49edbd4746ac36c92674ce0524; no voters: 04f29f2996a546f79797e6e6920a700f
21:22:24.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:24.081666 11517 raft_consensus.cc:2804] T 00000000000000000000000000000000 P f7480f49edbd4746ac36c92674ce0524 [term 1 FOLLOWER]: Leader pre-election won for term 2
21:22:24.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:24.082022 11517 raft_consensus.cc:493] T 00000000000000000000000000000000 P f7480f49edbd4746ac36c92674ce0524 [term 1 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
21:22:24.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:24.082340 11517 raft_consensus.cc:3060] T 00000000000000000000000000000000 P f7480f49edbd4746ac36c92674ce0524 [term 1 FOLLOWER]: Advancing to term 2
21:22:24.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:24.091992 11517 raft_consensus.cc:515] T 00000000000000000000000000000000 P f7480f49edbd4746ac36c92674ce0524 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "04f29f2996a546f79797e6e6920a700f" member_type: VOTER last_known_addr { host: "127.10.186.62" port: 46125 } } peers { permanent_uuid: "5ada6348d26f452da1cd3c3f9f9294f4" member_type: VOTER last_known_addr { host: "127.10.186.61" port: 45819 } } peers { permanent_uuid: "f7480f49edbd4746ac36c92674ce0524" member_type: VOTER last_known_addr { host: "127.10.186.60" port: 38107 } }
21:22:24.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:24.094043 11517 leader_election.cc:290] T 00000000000000000000000000000000 P f7480f49edbd4746ac36c92674ce0524 [CANDIDATE]: Term 2 election: Requested vote from peers 04f29f2996a546f79797e6e6920a700f (127.10.186.62:46125), 5ada6348d26f452da1cd3c3f9f9294f4 (127.10.186.61:45819)
21:22:24.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:24.096030 11461 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "f7480f49edbd4746ac36c92674ce0524" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "5ada6348d26f452da1cd3c3f9f9294f4"
21:22:24.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:24.096694 11461 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 5ada6348d26f452da1cd3c3f9f9294f4 [term 1 FOLLOWER]: Advancing to term 2
21:22:24.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:24.098333 11347 leader_election.cc:336] T 00000000000000000000000000000000 P f7480f49edbd4746ac36c92674ce0524 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 04f29f2996a546f79797e6e6920a700f (127.10.186.62:46125): Network error: Client connection negotiation failed: client connection to 127.10.186.62:46125: connect: Connection refused (error 111)
21:22:24.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:24.107999 11461 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 5ada6348d26f452da1cd3c3f9f9294f4 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate f7480f49edbd4746ac36c92674ce0524 in term 2.
21:22:24.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:24.110036 11347 leader_election.cc:304] T 00000000000000000000000000000000 P f7480f49edbd4746ac36c92674ce0524 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 5ada6348d26f452da1cd3c3f9f9294f4, f7480f49edbd4746ac36c92674ce0524; no voters: 04f29f2996a546f79797e6e6920a700f
21:22:24.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:24.111946 11517 raft_consensus.cc:2804] T 00000000000000000000000000000000 P f7480f49edbd4746ac36c92674ce0524 [term 2 FOLLOWER]: Leader election won for term 2
21:22:24.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:24.113797 11517 raft_consensus.cc:697] T 00000000000000000000000000000000 P f7480f49edbd4746ac36c92674ce0524 [term 2 LEADER]: Becoming Leader. State: Replica: f7480f49edbd4746ac36c92674ce0524, State: Running, Role: LEADER
21:22:24.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:24.114769 11517 consensus_queue.cc:237] T 00000000000000000000000000000000 P f7480f49edbd4746ac36c92674ce0524 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "04f29f2996a546f79797e6e6920a700f" member_type: VOTER last_known_addr { host: "127.10.186.62" port: 46125 } } peers { permanent_uuid: "5ada6348d26f452da1cd3c3f9f9294f4" member_type: VOTER last_known_addr { host: "127.10.186.61" port: 45819 } } peers { permanent_uuid: "f7480f49edbd4746ac36c92674ce0524" member_type: VOTER last_known_addr { host: "127.10.186.60" port: 38107 } }
21:22:24.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:24.124655 11522 sys_catalog.cc:455] T 00000000000000000000000000000000 P f7480f49edbd4746ac36c92674ce0524 [sys.catalog]: SysCatalogTable state changed. Reason: New leader f7480f49edbd4746ac36c92674ce0524. Latest consensus state: current_term: 2 leader_uuid: "f7480f49edbd4746ac36c92674ce0524" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "04f29f2996a546f79797e6e6920a700f" member_type: VOTER last_known_addr { host: "127.10.186.62" port: 46125 } } peers { permanent_uuid: "5ada6348d26f452da1cd3c3f9f9294f4" member_type: VOTER last_known_addr { host: "127.10.186.61" port: 45819 } } peers { permanent_uuid: "f7480f49edbd4746ac36c92674ce0524" member_type: VOTER last_known_addr { host: "127.10.186.60" port: 38107 } } }
21:22:24.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:24.125566 11522 sys_catalog.cc:458] T 00000000000000000000000000000000 P f7480f49edbd4746ac36c92674ce0524 [sys.catalog]: This master's current role is: LEADER
21:22:24.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:24.128300 11524 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
21:22:24.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:24.136345 11524 catalog_manager.cc:1494] Initializing Kudu cluster ID...
21:22:24.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:24.140421 11524 catalog_manager.cc:1269] Loaded cluster ID: 3e7dc32b40b94de0a90f622e8e1e29df
21:22:24.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:24.140858 11524 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
21:22:24.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:24.145438 11524 catalog_manager.cc:1514] Loading token signing keys...
21:22:24.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:24.150050 11524 catalog_manager.cc:6038] T 00000000000000000000000000000000 P f7480f49edbd4746ac36c92674ce0524: Loaded TSK: 0
21:22:24.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:24.151546 11524 catalog_manager.cc:1524] Initializing in-progress tserver states...
21:22:24.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:24.551120 11461 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 5ada6348d26f452da1cd3c3f9f9294f4 [term 2 FOLLOWER]: Refusing update from remote peer f7480f49edbd4746ac36c92674ce0524: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 2 index: 5. (index mismatch)
21:22:24.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:24.552860 11522 consensus_queue.cc:1048] T 00000000000000000000000000000000 P f7480f49edbd4746ac36c92674ce0524 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5ada6348d26f452da1cd3c3f9f9294f4" member_type: VOTER last_known_addr { host: "127.10.186.61" port: 45819 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
21:22:24.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:24.567751 11525 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5ada6348d26f452da1cd3c3f9f9294f4 [sys.catalog]: SysCatalogTable state changed. Reason: New leader f7480f49edbd4746ac36c92674ce0524. Latest consensus state: current_term: 2 leader_uuid: "f7480f49edbd4746ac36c92674ce0524" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "04f29f2996a546f79797e6e6920a700f" member_type: VOTER last_known_addr { host: "127.10.186.62" port: 46125 } } peers { permanent_uuid: "5ada6348d26f452da1cd3c3f9f9294f4" member_type: VOTER last_known_addr { host: "127.10.186.61" port: 45819 } } peers { permanent_uuid: "f7480f49edbd4746ac36c92674ce0524" member_type: VOTER last_known_addr { host: "127.10.186.60" port: 38107 } } }
21:22:24.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:24.570287 11525 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5ada6348d26f452da1cd3c3f9f9294f4 [sys.catalog]: This master's current role is: FOLLOWER
21:22:24.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:24.578456 11525 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5ada6348d26f452da1cd3c3f9f9294f4 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "f7480f49edbd4746ac36c92674ce0524" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "04f29f2996a546f79797e6e6920a700f" member_type: VOTER last_known_addr { host: "127.10.186.62" port: 46125 } } peers { permanent_uuid: "5ada6348d26f452da1cd3c3f9f9294f4" member_type: VOTER last_known_addr { host: "127.10.186.61" port: 45819 } } peers { permanent_uuid: "f7480f49edbd4746ac36c92674ce0524" member_type: VOTER last_known_addr { host: "127.10.186.60" port: 38107 } } }
21:22:24.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:24.579438 11525 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5ada6348d26f452da1cd3c3f9f9294f4 [sys.catalog]: This master's current role is: FOLLOWER
21:22:24.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:24.586802 11347 consensus_peers.cc:597] T 00000000000000000000000000000000 P f7480f49edbd4746ac36c92674ce0524 -> Peer 04f29f2996a546f79797e6e6920a700f (127.10.186.62:46125): Couldn't send request to peer 04f29f2996a546f79797e6e6920a700f. Status: Network error: Client connection negotiation failed: client connection to 127.10.186.62:46125: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
21:22:24.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:24.588587 11529 sys_catalog.cc:455] T 00000000000000000000000000000000 P f7480f49edbd4746ac36c92674ce0524 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "f7480f49edbd4746ac36c92674ce0524" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "04f29f2996a546f79797e6e6920a700f" member_type: VOTER last_known_addr { host: "127.10.186.62" port: 46125 } } peers { permanent_uuid: "5ada6348d26f452da1cd3c3f9f9294f4" member_type: VOTER last_known_addr { host: "127.10.186.61" port: 45819 } } peers { permanent_uuid: "f7480f49edbd4746ac36c92674ce0524" member_type: VOTER last_known_addr { host: "127.10.186.60" port: 38107 } } }
21:22:24.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:24.589347 11529 sys_catalog.cc:458] T 00000000000000000000000000000000 P f7480f49edbd4746ac36c92674ce0524 [sys.catalog]: This master's current role is: LEADER
21:22:24.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:23.562682 11512 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:22:24.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:23.564363 11514 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:22:24.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:24.772524 11513 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
21:22:24.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:24.773126 11488 server_base.cc:1055] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:22:24.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:24.777604 11488 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:22:24.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:24.780771 11488 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:22:24.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:24.782192 11488 hybrid_clock.cc:648] HybridClock initialized: now 1769635344782155 us; error 54 us; skew 500 ppm
21:22:24.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:24.783088 11488 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:22:24.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 28 21:22:24 dist-test-slave-hq3s krb5kdc[10996](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1769635344, etypes {rep=17 tkt=17 ses=17}, kudu/127.10.186.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:22:24.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:24.789184 11488 init.cc:377] Logged in from keytab as kudu/127.10.186.62@KRBTEST.COM (short username kudu)
21:22:24.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:24.794366 11488 webserver.cc:492] Webserver started at http://127.10.186.62:37867/ using document root <none> and password file <none>
21:22:24.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:24.795305 11488 fs_manager.cc:362] Metadata directory not provided
21:22:24.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:24.795542 11488 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:22:24.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:24.804104 11488 fs_manager.cc:714] Time spent opening directory manager: real 0.006s user 0.007s sys 0.000s
21:22:24.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:24.809634 11535 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:22:24.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:24.810962 11488 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.006s sys 0.000s
21:22:24.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:24.811309 11488 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/master-0/data,/tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/master-0/wal
21:22:24.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "04f29f2996a546f79797e6e6920a700f"
21:22:24.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-01-28 21:22:11 on dist-test-slave-hq3s"
21:22:24.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:24.813297 11488 fs_report.cc:389] FS layout report
21:22:24.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:22:24.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/master-0/wal
21:22:24.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/master-0/wal
21:22:24.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/master-0/data/data
21:22:24.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:22:24.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:22:24.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:22:24.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:22:24.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:22:24.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:22:24.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:22:24.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:22:24.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:22:24.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:22:24.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:22:24.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:24.859335 11488 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:22:24.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:24.864815 11488 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:22:24.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:24.865296 11488 kserver.cc:163] Server-wide thread pool size limit: 3276
21:22:24.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:24.955408 11488 rpc_server.cc:307] RPC server started. Bound to: 127.10.186.62:46125
21:22:24.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:24.955471 11586 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.10.186.62:46125 every 8 connection(s)
21:22:24.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:24.958217 11488 server_base.cc:1192] Dumped server information to /tmp/dist-test-taskp7EThw/test-tmp/mini-kudu-cluster2141602027260210637/master-0/data/info.pb
21:22:24.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:24.961536 10984 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskp7EThw/build/tsan/bin/kudu as pid 11488
21:22:24.962 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:22:24.963 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.10.186.60:38107(127.10.186.60:38107)] connecting to peer
21:22:24.964 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.10.186.61:45819(127.10.186.61:45819)] connecting to peer
21:22:24.966 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.10.186.62:46125(127.10.186.62:46125)] connecting to peer
21:22:24.972 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.10.186.60:38107(127.10.186.60:38107)] Successfully connected to peer
21:22:24.973 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.10.186.62:46125(127.10.186.62:46125)] Successfully connected to peer
21:22:24.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:24.975509 11587 sys_catalog.cc:263] Verifying existing consensus state
21:22:24.977 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.10.186.61:45819(127.10.186.61:45819)] Successfully connected to peer
21:22:24.981 [WARN - kudu-nio-0] (SecurityContext.java:206) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
21:22:24.983 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.10.186.60
21:22:24.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:24.983879 11587 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 04f29f2996a546f79797e6e6920a700f: Bootstrap starting.
21:22:25.012 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.186.60:38107(127.10.186.60:38107)] cleaning up while in state NEGOTIATING due to: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
21:22:25.012 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.186.60:38107: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
21:22:25.013 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.10.186.60:38107(127.10.186.60:38107)] handling channelInactive
21:22:25.015 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.186.61:45819(127.10.186.61:45819)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
21:22:25.016 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.186.61:45819: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
21:22:25.016 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.10.186.61:45819(127.10.186.61:45819)] handling channelInactive
21:22:25.018 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.186.62:46125(127.10.186.62:46125)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
21:22:25.019 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.186.62:46125: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
21:22:25.019 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1523) Notify RPC KuduRpc(method=ListTabletServers, tablet=null, attempt=1, TimeoutTracker(timeout=30000, elapsed=57), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.186.60:38107, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.186.61:45819, [3ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.186.62:46125, [50ms] Sub RPC ConnectToMaster: received response from server master-127.10.186.60:38107: Configuration error: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), [54ms] Sub RPC ConnectToMaster: received response from server master-127.10.186.61:45819: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [57ms] Sub RPC ConnectToMaster: received response from server master-127.10.186.62:46125: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, deferred=null) after lookup exception
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.10.186.60:38107,127.10.186.61:45819,127.10.186.62:46125). Exceptions received: [org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268) [main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) [main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387) [main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376) [main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) [async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) [async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315) [async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286) [async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) [async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) [async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) [main/:?]
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331) [main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256) [main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) [main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) [main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) [main/:?]
at org.apache.kudu.client.Connection.cleanup(Connection.java:737) [main/:?]
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455) [main/:?]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) [netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) [netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) [netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) [netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) [netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_275]
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.10.186.60:38107,127.10.186.61:45819,127.10.186.62:46125). Exceptions received: [org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
21:22:25.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:25.023656 11592 negotiation.cc:337] Failed RPC negotiation. Trace:
21:22:25.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0128 21:22:24.967234 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:43982 (local address 127.10.186.60:38107)
21:22:25.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0128 21:22:24.980108 (+ 12874us) server_negotiation.cc:207] Beginning negotiation
21:22:25.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0128 21:22:24.980139 (+ 31us) server_negotiation.cc:400] Waiting for connection header
21:22:25.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0128 21:22:24.980196 (+ 57us) server_negotiation.cc:408] Connection header received
21:22:25.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0128 21:22:24.980351 (+ 155us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
21:22:25.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0128 21:22:24.980380 (+ 29us) server_negotiation.cc:462] Received NEGOTIATE request from client
21:22:25.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0128 21:22:24.980748 (+ 368us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
21:22:25.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0128 21:22:24.989223 (+ 8475us) server_negotiation.cc:227] Negotiated authn=SASL
21:22:25.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0128 21:22:24.989773 (+ 550us) server_negotiation.cc:366] Received TLS_HANDSHAKE NegotiatePB request
21:22:25.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0128 21:22:24.991811 (+ 2038us) server_negotiation.cc:378] Sending TLS_HANDSHAKE NegotiatePB response
21:22:25.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0128 21:22:25.010183 (+ 18372us) server_negotiation.cc:366] Received TLS_HANDSHAKE NegotiatePB request
21:22:25.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0128 21:22:25.010853 (+ 670us) server_negotiation.cc:663] Negotiated TLSv1.3 with cipher TLS_AES_128_GCM_SHA256 TLSv1.3 Kx=any Au=any Enc=AESGCM(128) Mac=AEAD
21:22:25.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0128 21:22:25.014272 (+ 3419us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:43982: BlockingRecv error: failed to read from TLS socket (remote: unknown): Cannot send after transport endpoint shutdown (error 108)
21:22:25.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":12203,"thread_start_us":11544,"threads_started":1}
21:22:25.025 [DEBUG - kudu-nio-0] (KuduRpc.java:260) Handling a callback on RPC KuduRpc(method=ListTabletServers, tablet=null, attempt=1, TimeoutTracker(timeout=30000, elapsed=60), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.186.60:38107, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.186.61:45819, [3ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.186.62:46125, [50ms] Sub RPC ConnectToMaster: received response from server master-127.10.186.60:38107: Configuration error: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), [54ms] Sub RPC ConnectToMaster: received response from server master-127.10.186.61:45819: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [57ms] Sub RPC ConnectToMaster: received response from server master-127.10.186.62:46125: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, deferred=null) with no deferred attached!
21:22:25.025 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.10.186.62:46125(127.10.186.62:46125)] disconnecting while in state TERMINATED
21:22:25.025 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.10.186.62:46125(127.10.186.62:46125)] handling channelInactive
21:22:25.025 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.10.186.61:45819(127.10.186.61:45819)] disconnecting while in state TERMINATED
21:22:25.026 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.10.186.60:38107(127.10.186.60:38107)] disconnecting while in state TERMINATED
21:22:25.027 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2704) Releasing all remaining resources
21:22:25.028 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
21:22:25.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:25.029104 11593 negotiation.cc:337] Failed RPC negotiation. Trace:
21:22:25.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0128 21:22:24.973318 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:49968 (local address 127.10.186.62:46125)
21:22:25.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0128 21:22:25.004033 (+ 30715us) server_negotiation.cc:207] Beginning negotiation
21:22:25.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0128 21:22:25.004083 (+ 50us) server_negotiation.cc:400] Waiting for connection header
21:22:25.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0128 21:22:25.004176 (+ 93us) server_negotiation.cc:408] Connection header received
21:22:25.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0128 21:22:25.015787 (+ 11611us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
21:22:25.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0128 21:22:25.015857 (+ 70us) server_negotiation.cc:462] Received NEGOTIATE request from client
21:22:25.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0128 21:22:25.016336 (+ 479us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
21:22:25.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0128 21:22:25.016818 (+ 482us) server_negotiation.cc:227] Negotiated authn=SASL
21:22:25.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0128 21:22:25.026897 (+ 10079us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:49968: BlockingRecv error: recv got EOF from 127.0.0.1:49968 (error 108)
21:22:25.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":29067,"thread_start_us":29122,"threads_started":1}
21:22:25.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260128 21:22:25.033749 11594 negotiation.cc:337] Failed RPC negotiation. Trace:
21:22:25.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0128 21:22:24.977132 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:36210 (local address 127.10.186.61:45819)
21:22:25.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0128 21:22:25.014698 (+ 37566us) server_negotiation.cc:207] Beginning negotiation
21:22:25.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0128 21:22:25.014727 (+ 29us) server_negotiation.cc:400] Waiting for connection header
21:22:25.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0128 21:22:25.014777 (+ 50us) server_negotiation.cc:408] Connection header received
21:22:25.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0128 21:22:25.014909 (+ 132us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
21:22:25.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0128 21:22:25.014935 (+ 26us) server_negotiation.cc:462] Received NEGOTIATE request from client
21:22:25.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0128 21:22:25.015219 (+ 284us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
21:22:25.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0128 21:22:25.015452 (+ 233us) server_negotiation.cc:227] Negotiated authn=SASL
21:22:25.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0128 21:22:25.032251 (+ 16799us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:36210: BlockingRecv error: recv got EOF from 127.0.0.1:36210 (error 108)
21:22:25.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":36947,"thread_start_us":38412,"threads_started":1}
21:22:25.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:25.044708 11587 log.cc:826] T 00000000000000000000000000000000 P 04f29f2996a546f79797e6e6920a700f: Log is configured to *not* fsync() on all Append() calls
21:22:25.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:25.046065 10984 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskp7EThw/build/tsan/bin/kudu with pid 11488
21:22:25.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:25.079778 10984 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskp7EThw/build/tsan/bin/kudu with pid 11398
21:22:25.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:25.101740 10984 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskp7EThw/build/tsan/bin/kudu with pid 11326
21:22:26.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-01-28T21:22:26Z chronyd exiting
21:22:27.240 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
21:22:27.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:27.241252 10261 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskp7EThw/build/tsan/bin/kudu with pid 10546
21:22:27.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:27.260304 10261 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskp7EThw/build/tsan/bin/kudu with pid 10697
21:22:27.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:27.279140 10261 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskp7EThw/build/tsan/bin/kudu with pid 10841
21:22:27.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:27.300710 10261 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskp7EThw/build/tsan/bin/kudu with pid 10273
21:22:27.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:27.326287 10261 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskp7EThw/build/tsan/bin/kudu with pid 10343
21:22:27.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260128 21:22:27.351239 10261 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskp7EThw/build/tsan/bin/kudu with pid 10418
21:22:28.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-01-28T21:22:28Z chronyd exiting
21:22:29.446 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectExpires: failed attempt 1
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.10.186.60:38107,127.10.186.61:45819,127.10.186.62:46125). Exceptions received: [org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available]
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity$3.get(TestSecurity.java:386) ~[test/:?]
at org.apache.kudu.test.junit.AssertHelpers.assertEventuallyTrue(AssertHelpers.java:40) ~[kudu-test-utils-1.19.0-SNAPSHOT-unshaded.jar:1.19.0-SNAPSHOT]
at org.apache.kudu.client.TestSecurity.assertEventualAuthenticationFailure(TestSecurity.java:378) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectExpires(TestSecurity.java:412) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.cleanup(Connection.java:737) ~[main/:?]
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455) ~[main/:?]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.10.186.60:38107,127.10.186.61:45819,127.10.186.62:46125). Exceptions received: [org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available]
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity$3.get(TestSecurity.java:386)
at org.apache.kudu.test.junit.AssertHelpers.assertEventuallyTrue(AssertHelpers.java:40)
at org.apache.kudu.client.TestSecurity.assertEventualAuthenticationFailure(TestSecurity.java:378)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectExpires(TestSecurity.java:412)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:748)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more