16:19:34.280 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
16:19:34.295 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-tasko3a_Hu/test-tmp
16:19:34.303 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
16:19:34.307 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-tasko3a_Hu/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
16:19:34.524 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster1397566977689697145"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
16:19:34.737 [DEBUG - main] (MiniKuduCluster.java:195) Response:
16:19:34.738 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
16:19:34.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:34.742980 21831 mini_oidc.cc:150] Starting JWKS server
16:19:34.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:34.746641 21831 webserver.cc:533] Webserver started at http://127.0.0.1:45665/ using document root <none> and password file <none>
16:19:34.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:34.747037 21831 mini_oidc.cc:186] Starting OIDC Discovery server
16:19:34.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:34.749006 21831 webserver.cc:533] Webserver started at http://127.0.0.1:43089/ using document root <none> and password file <none>
16:19:34.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-23T16:19:34Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
16:19:34.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-23T16:19:34Z Disabled control of system clock
16:19:34.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:34.809288 21831 external_mini_cluster.cc:1372] Running /tmp/dist-test-tasko3a_Hu/build/tsan/bin/kudu
16:19:34.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasko3a_Hu/build/tsan/bin/kudu
16:19:34.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster1397566977689697145/master-0/wal
16:19:34.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster1397566977689697145/master-0/data
16:19:34.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:19:34.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:19:34.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:19:34.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:19:34.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:19:34.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:19:34.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster1397566977689697145/master-0/logs
16:19:34.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster1397566977689697145/master-0/data/info.pb
16:19:34.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:19:34.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:19:34.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:19:34.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:19:34.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:19:34.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:19:34.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:19:34.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:19:34.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
16:19:34.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:19:34.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:19:34.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:19:34.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.81.254:40179
16:19:34.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.81.254
16:19:34.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:19:34.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.81.212:37463
16:19:34.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:19:34.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:19:34.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:19:34.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:19:34.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.21.81.254:40179,127.21.81.253:41791,127.21.81.252:37327
16:19:34.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:19:34.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45665/jwks.json with env {}
16:19:35.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:35.124693 21843 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
16:19:35.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:35.125401 21843 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:19:35.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:35.126222 21843 flags.cc:432] Enabled unsafe flag: --never_fsync=true
16:19:35.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:35.159964 21843 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
16:19:35.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:35.160341 21843 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
16:19:35.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:35.160658 21843 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
16:19:35.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:35.160995 21843 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
16:19:35.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:35.161355 21843 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
16:19:35.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:35.161618 21843 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:45665/jwks.json
16:19:35.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:35.198434 21843 master_runner.cc:387] Master server non-default flags:
16:19:35.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster1397566977689697145/master-0/logs
16:19:35.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:19:35.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:19:35.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:19:35.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.81.212:37463
16:19:35.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:19:35.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:19:35.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster1397566977689697145/master-0/data
16:19:35.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster1397566977689697145/master-0/wal
16:19:35.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:19:35.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.21.81.254:40179,127.21.81.253:41791,127.21.81.252:37327
16:19:35.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:19:35.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:19:35.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:19:35.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.81.254:40179
16:19:35.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:19:35.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:19:35.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:19:35.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45665/jwks.json
16:19:35.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:19:35.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:19:35.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster1397566977689697145/master-0/data/info.pb
16:19:35.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.81.254
16:19:35.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:19:35.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:19:35.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:19:35.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:19:35.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:19:35.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:19:35.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:19:35.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
16:19:35.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
16:19:35.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision faa920b8364ca43fd9d419253dab85c46533dc98
16:19:35.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:19:35.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 23 Sep 2026 15:47:26 UTC on e7f111948823
16:19:35.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 13962
16:19:35.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:19:35.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:35.200198 21843 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:19:35.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:35.202427 21843 file_cache.cc:504] Constructed file cache file cache with capacity 419430
16:19:35.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:35.217641 21849 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:19:36.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:35.218362 21850 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:19:36.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:36.418335 21843 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 1.202s user 0.471s sys 0.730s
16:19:36.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:36.418749 21843 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 1.203s user 0.471s sys 0.730s
16:19:36.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:36.421356 21843 server_base.cc:1061] running on GCE node
16:19:36.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:36.423133 21854 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:19:36.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:36.424412 21843 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:19:36.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:36.426930 21843 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:19:36.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:36.428337 21843 hybrid_clock.cc:648] HybridClock initialized: now 1790180376428282 us; error 51 us; skew 500 ppm
16:19:36.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:36.429365 21843 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:19:36.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:36.438627 21843 webserver.cc:533] Webserver started at http://127.21.81.254:46695/ using document root <none> and password file <none>
16:19:36.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:36.439733 21843 fs_manager.cc:362] Metadata directory not provided
16:19:36.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:36.439996 21843 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:19:36.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:36.440595 21843 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
16:19:36.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:36.444939 21843 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster1397566977689697145/master-0/data/instance:
16:19:36.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1b2d44848607499bb0e56d37b3328000"
16:19:36.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-23 16:19:36 on dist-test-slave-fqp8"
16:19:36.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:36.446448 21843 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster1397566977689697145/master-0/wal/instance:
16:19:36.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1b2d44848607499bb0e56d37b3328000"
16:19:36.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-23 16:19:36 on dist-test-slave-fqp8"
16:19:36.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:36.454942 21843 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.002s sys 0.006s
16:19:36.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:36.461292 21859 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:19:36.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:36.462812 21843 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.002s sys 0.001s
16:19:36.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:36.463225 21843 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster1397566977689697145/master-0/data,/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster1397566977689697145/master-0/wal
16:19:36.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1b2d44848607499bb0e56d37b3328000"
16:19:36.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-23 16:19:36 on dist-test-slave-fqp8"
16:19:36.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:36.463653 21843 fs_report.cc:389] FS layout report
16:19:36.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:19:36.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster1397566977689697145/master-0/wal
16:19:36.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster1397566977689697145/master-0/wal
16:19:36.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster1397566977689697145/master-0/data/data
16:19:36.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:19:36.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:19:36.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:19:36.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:19:36.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:19:36.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:19:36.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:19:36.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:19:36.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:19:36.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:19:36.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:19:36.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:36.534454 21843 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
16:19:36.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:36.535856 21843 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:19:36.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:36.536258 21843 kserver.cc:163] Server-wide thread pool size limit: 3276
16:19:36.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:36.607452 21843 rpc_server.cc:307] RPC server started. Bound to: 127.21.81.254:40179
16:19:36.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:36.607539 21910 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.21.81.254:40179 every 8 connection(s)
16:19:36.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:36.610702 21843 server_base.cc:1193] Dumped server information to /tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster1397566977689697145/master-0/data/info.pb
16:19:36.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:36.614753 21831 external_mini_cluster.cc:1434] Started /tmp/dist-test-tasko3a_Hu/build/tsan/bin/kudu as pid 21843
16:19:36.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:36.615275 21831 external_mini_cluster.cc:1448] Reading /tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster1397566977689697145/master-0/wal/instance
16:19:36.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:36.617935 21911 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
16:19:36.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:36.620287 21831 external_mini_cluster.cc:1372] Running /tmp/dist-test-tasko3a_Hu/build/tsan/bin/kudu
16:19:36.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasko3a_Hu/build/tsan/bin/kudu
16:19:36.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster1397566977689697145/master-1/wal
16:19:36.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster1397566977689697145/master-1/data
16:19:36.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:19:36.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:19:36.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:19:36.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:19:36.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:19:36.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:19:36.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster1397566977689697145/master-1/logs
16:19:36.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster1397566977689697145/master-1/data/info.pb
16:19:36.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:19:36.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:19:36.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:19:36.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:19:36.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:19:36.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:19:36.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:19:36.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:19:36.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
16:19:36.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:19:36.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:19:36.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:19:36.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.81.253:41791
16:19:36.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.81.253
16:19:36.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:19:36.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.81.212:37463
16:19:36.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:19:36.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:19:36.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:19:36.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:19:36.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.21.81.254:40179,127.21.81.253:41791,127.21.81.252:37327
16:19:36.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:19:36.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45665/jwks.json with env {}
16:19:36.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:36.640522 21911 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.21.81.254" port: 40179 } has no permanent_uuid. Determining permanent_uuid...
16:19:36.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:36.667431 21911 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.21.81.253" port: 41791 } has no permanent_uuid. Determining permanent_uuid...
16:19:36.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:36.669937 21863 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.21.81.253:41791: connect: Connection refused (error 111)
16:19:36.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:36.674060 21911 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.21.81.253:41791: Network error: Client connection negotiation failed: client connection to 127.21.81.253:41791: connect: Connection refused (error 111)
16:19:36.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:36.723610 21911 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.81.253" port: 41791 } attempt: 1
16:19:36.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:36.728118 21911 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.21.81.253:41791: Network error: Client connection negotiation failed: client connection to 127.21.81.253:41791: connect: Connection refused (error 111)
16:19:36.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:36.796599 21911 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.81.253" port: 41791 } attempt: 2
16:19:36.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:36.801110 21911 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.21.81.253:41791: Network error: Client connection negotiation failed: client connection to 127.21.81.253:41791: connect: Connection refused (error 111)
16:19:36.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:36.892568 21911 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.81.253" port: 41791 } attempt: 3
16:19:36.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:36.899605 21911 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.21.81.253:41791: Network error: Client connection negotiation failed: client connection to 127.21.81.253:41791: connect: Connection refused (error 111)
16:19:36.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:36.933033 21913 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
16:19:36.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:36.933715 21913 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:19:36.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:36.934558 21913 flags.cc:432] Enabled unsafe flag: --never_fsync=true
16:19:36.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:36.967348 21913 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
16:19:36.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:36.967726 21913 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
16:19:36.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:36.968057 21913 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
16:19:36.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:36.968403 21913 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
16:19:36.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:36.968768 21913 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
16:19:36.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:36.969055 21913 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:45665/jwks.json
16:19:37.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:37.007411 21913 master_runner.cc:387] Master server non-default flags:
16:19:37.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster1397566977689697145/master-1/logs
16:19:37.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:19:37.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:19:37.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:19:37.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.81.212:37463
16:19:37.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:19:37.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:19:37.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster1397566977689697145/master-1/data
16:19:37.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster1397566977689697145/master-1/wal
16:19:37.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:19:37.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.21.81.254:40179,127.21.81.253:41791,127.21.81.252:37327
16:19:37.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:19:37.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:19:37.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:19:37.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.81.253:41791
16:19:37.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:19:37.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:19:37.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:19:37.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45665/jwks.json
16:19:37.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:19:37.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:19:37.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster1397566977689697145/master-1/data/info.pb
16:19:37.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.81.253
16:19:37.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:19:37.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:19:37.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:19:37.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:19:37.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:19:37.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:19:37.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:19:37.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
16:19:37.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
16:19:37.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision faa920b8364ca43fd9d419253dab85c46533dc98
16:19:37.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:19:37.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 23 Sep 2026 15:47:26 UTC on e7f111948823
16:19:37.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 13962
16:19:37.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:19:37.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:37.009126 21913 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:19:37.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:37.011446 21913 file_cache.cc:504] Constructed file cache file cache with capacity 419430
16:19:37.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:37.023329 21922 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:19:37.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:37.043005 21911 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.81.253" port: 41791 } attempt: 4
16:19:37.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:37.047866 21911 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.21.81.253:41791: Network error: Client connection negotiation failed: client connection to 127.21.81.253:41791: connect: Connection refused (error 111)
16:19:37.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:37.347277 21911 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.81.253" port: 41791 } attempt: 5
16:19:37.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:37.354276 21911 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.21.81.253:41791: Network error: Client connection negotiation failed: client connection to 127.21.81.253:41791: connect: Connection refused (error 111)
16:19:37.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:37.901824 21911 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.81.253" port: 41791 } attempt: 6
16:19:37.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:37.917577 21911 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.21.81.253:41791: Network error: Client connection negotiation failed: client connection to 127.21.81.253:41791: connect: Connection refused (error 111)
16:19:38.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:37.025851 21923 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:19:38.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:37.028533 21925 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:19:38.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:38.260798 21924 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1230 milliseconds
16:19:38.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:38.260912 21913 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
16:19:38.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:38.262380 21913 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:19:38.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:38.265509 21913 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:19:38.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:38.266949 21913 hybrid_clock.cc:648] HybridClock initialized: now 1790180378266909 us; error 58 us; skew 500 ppm
16:19:38.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:38.267709 21913 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:19:38.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:38.273595 21913 webserver.cc:533] Webserver started at http://127.21.81.253:46121/ using document root <none> and password file <none>
16:19:38.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:38.274510 21913 fs_manager.cc:362] Metadata directory not provided
16:19:38.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:38.274694 21913 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:19:38.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:38.275115 21913 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
16:19:38.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:38.279215 21913 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster1397566977689697145/master-1/data/instance:
16:19:38.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "04b233e6e07044f2a4fbf857e0e0c4f8"
16:19:38.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-23 16:19:38 on dist-test-slave-fqp8"
16:19:38.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:38.280310 21913 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster1397566977689697145/master-1/wal/instance:
16:19:38.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "04b233e6e07044f2a4fbf857e0e0c4f8"
16:19:38.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-23 16:19:38 on dist-test-slave-fqp8"
16:19:38.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:38.287345 21913 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.000s sys 0.008s
16:19:38.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:38.292925 21934 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:19:38.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:38.294231 21913 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.000s
16:19:38.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:38.294544 21913 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster1397566977689697145/master-1/data,/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster1397566977689697145/master-1/wal
16:19:38.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "04b233e6e07044f2a4fbf857e0e0c4f8"
16:19:38.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-23 16:19:38 on dist-test-slave-fqp8"
16:19:38.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:38.294827 21913 fs_report.cc:389] FS layout report
16:19:38.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:19:38.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster1397566977689697145/master-1/wal
16:19:38.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster1397566977689697145/master-1/wal
16:19:38.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster1397566977689697145/master-1/data/data
16:19:38.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:19:38.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:19:38.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:19:38.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:19:38.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:19:38.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:19:38.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:19:38.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:19:38.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:19:38.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:19:38.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:19:38.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:38.340153 21913 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
16:19:38.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:38.341593 21913 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:19:38.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:38.342026 21913 kserver.cc:163] Server-wide thread pool size limit: 3276
16:19:38.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:38.418864 21913 rpc_server.cc:307] RPC server started. Bound to: 127.21.81.253:41791
16:19:38.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:38.418938 21985 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.21.81.253:41791 every 8 connection(s)
16:19:38.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:38.421772 21913 server_base.cc:1193] Dumped server information to /tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster1397566977689697145/master-1/data/info.pb
16:19:38.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:38.427382 21986 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
16:19:38.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:38.430300 21831 external_mini_cluster.cc:1434] Started /tmp/dist-test-tasko3a_Hu/build/tsan/bin/kudu as pid 21913
16:19:38.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:38.430757 21831 external_mini_cluster.cc:1448] Reading /tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster1397566977689697145/master-1/wal/instance
16:19:38.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:38.433637 21831 external_mini_cluster.cc:1372] Running /tmp/dist-test-tasko3a_Hu/build/tsan/bin/kudu
16:19:38.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasko3a_Hu/build/tsan/bin/kudu
16:19:38.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster1397566977689697145/master-2/wal
16:19:38.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster1397566977689697145/master-2/data
16:19:38.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:19:38.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:19:38.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:19:38.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:19:38.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:19:38.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:19:38.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster1397566977689697145/master-2/logs
16:19:38.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster1397566977689697145/master-2/data/info.pb
16:19:38.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:19:38.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:19:38.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:19:38.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:19:38.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:19:38.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:19:38.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:19:38.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:19:38.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
16:19:38.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:19:38.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:19:38.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:19:38.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.81.252:37327
16:19:38.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.81.252
16:19:38.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:19:38.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.81.212:37463
16:19:38.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:19:38.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:19:38.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:19:38.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:19:38.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.21.81.254:40179,127.21.81.253:41791,127.21.81.252:37327
16:19:38.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:19:38.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45665/jwks.json with env {}
16:19:38.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:38.441706 21986 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.21.81.254" port: 40179 } has no permanent_uuid. Determining permanent_uuid...
16:19:38.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:38.462813 21986 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.21.81.253" port: 41791 } has no permanent_uuid. Determining permanent_uuid...
16:19:38.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:38.477706 21986 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.21.81.252" port: 37327 } has no permanent_uuid. Determining permanent_uuid...
16:19:38.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:38.480067 21938 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.21.81.252:37327: connect: Connection refused (error 111)
16:19:38.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:38.484395 21986 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.21.81.252:37327: Network error: Client connection negotiation failed: client connection to 127.21.81.252:37327: connect: Connection refused (error 111)
16:19:38.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:38.533987 21986 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.81.252" port: 37327 } attempt: 1
16:19:38.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:38.538283 21986 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.21.81.252:37327: Network error: Client connection negotiation failed: client connection to 127.21.81.252:37327: connect: Connection refused (error 111)
16:19:38.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:38.606743 21986 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.81.252" port: 37327 } attempt: 2
16:19:38.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:38.612061 21986 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.21.81.252:37327: Network error: Client connection negotiation failed: client connection to 127.21.81.252:37327: connect: Connection refused (error 111)
16:19:38.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:38.703488 21986 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.81.252" port: 37327 } attempt: 3
16:19:38.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:38.708094 21986 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.21.81.252:37327: Network error: Client connection negotiation failed: client connection to 127.21.81.252:37327: connect: Connection refused (error 111)
16:19:38.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:38.747255 21988 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
16:19:38.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:38.747929 21988 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:19:38.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:38.748601 21988 flags.cc:432] Enabled unsafe flag: --never_fsync=true
16:19:38.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:38.781270 21988 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
16:19:38.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:38.781637 21988 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
16:19:38.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:38.782006 21988 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
16:19:38.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:38.782374 21988 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
16:19:38.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:38.782759 21988 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
16:19:38.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:38.783074 21988 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:45665/jwks.json
16:19:38.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:38.820150 21988 master_runner.cc:387] Master server non-default flags:
16:19:38.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster1397566977689697145/master-2/logs
16:19:38.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:19:38.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:19:38.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:19:38.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.81.212:37463
16:19:38.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:19:38.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:19:38.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster1397566977689697145/master-2/data
16:19:38.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster1397566977689697145/master-2/wal
16:19:38.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:19:38.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.21.81.254:40179,127.21.81.253:41791,127.21.81.252:37327
16:19:38.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:19:38.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:19:38.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:19:38.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.81.252:37327
16:19:38.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:19:38.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:19:38.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:19:38.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45665/jwks.json
16:19:38.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:19:38.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:19:38.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster1397566977689697145/master-2/data/info.pb
16:19:38.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.81.252
16:19:38.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:19:38.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:19:38.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:19:38.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:19:38.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:19:38.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:19:38.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:19:38.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
16:19:38.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
16:19:38.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision faa920b8364ca43fd9d419253dab85c46533dc98
16:19:38.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:19:38.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 23 Sep 2026 15:47:26 UTC on e7f111948823
16:19:38.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 13962
16:19:38.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:19:38.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:38.821906 21988 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:19:38.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:38.824223 21988 file_cache.cc:504] Constructed file cache file cache with capacity 419430
16:19:38.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:38.836352 21998 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:19:38.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:38.851487 21986 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.81.252" port: 37327 } attempt: 4
16:19:38.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:38.860455 21986 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.21.81.252:37327: Network error: Client connection negotiation failed: client connection to 127.21.81.252:37327: connect: Connection refused (error 111)
16:19:38.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:38.978235 21911 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.81.253" port: 41791 } attempt: 7
16:19:39.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:39.026994 21911 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.21.81.252" port: 37327 } has no permanent_uuid. Determining permanent_uuid...
16:19:39.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:39.062765 21911 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.21.81.252:37327: Network error: Client connection negotiation failed: client connection to 127.21.81.252:37327: connect: Connection refused (error 111)
16:19:39.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:39.121436 21911 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.81.252" port: 37327 } attempt: 1
16:19:39.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:39.131961 21911 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.21.81.252:37327: Network error: Client connection negotiation failed: client connection to 127.21.81.252:37327: connect: Connection refused (error 111)
16:19:39.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:39.160022 21986 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.81.252" port: 37327 } attempt: 5
16:19:39.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:39.170195 21986 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.21.81.252:37327: Network error: Client connection negotiation failed: client connection to 127.21.81.252:37327: connect: Connection refused (error 111)
16:19:39.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:39.213631 21911 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.81.252" port: 37327 } attempt: 2
16:19:39.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:39.223981 21911 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.21.81.252:37327: Network error: Client connection negotiation failed: client connection to 127.21.81.252:37327: connect: Connection refused (error 111)
16:19:39.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:39.309616 21911 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.81.252" port: 37327 } attempt: 3
16:19:39.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:39.318490 21911 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.21.81.252:37327: Network error: Client connection negotiation failed: client connection to 127.21.81.252:37327: connect: Connection refused (error 111)
16:19:39.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:39.459010 21911 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.81.252" port: 37327 } attempt: 4
16:19:39.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:39.467383 21911 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.21.81.252:37327: Network error: Client connection negotiation failed: client connection to 127.21.81.252:37327: connect: Connection refused (error 111)
16:19:39.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:39.717849 21986 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.81.252" port: 37327 } attempt: 6
16:19:39.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:39.750593 21986 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.21.81.252:37327: Network error: Client connection negotiation failed: client connection to 127.21.81.252:37327: connect: Connection refused (error 111)
16:19:39.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:39.750919 21911 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.81.252" port: 37327 } attempt: 5
16:19:39.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:39.759027 21911 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.21.81.252:37327: Network error: Client connection negotiation failed: client connection to 127.21.81.252:37327: connect: Connection refused (error 111)
16:19:40.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:40.311554 21911 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.81.252" port: 37327 } attempt: 6
16:19:40.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:40.343556 21911 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.21.81.252:37327: Network error: Client connection negotiation failed: client connection to 127.21.81.252:37327: connect: Connection refused (error 111)
16:19:40.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:40.244876 21997 debug-util.cc:398] Leaking SignalData structure 0x7b08000379c0 after lost signal to thread 21988
16:19:40.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:40.531903 21997 kernel_stack_watchdog.cc:198] Thread 21988 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 404ms:
16:19:40.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
16:19:40.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
16:19:40.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
16:19:40.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
16:19:40.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:38.837751 21999 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:19:40.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:40.532894 21988 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.695s user 0.629s sys 1.040s
16:19:40.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:40.534242 21988 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.696s user 0.630s sys 1.040s
16:19:40.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:40.535888 22001 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:19:40.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:40.537544 22000 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1694 milliseconds
16:19:40.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:40.537593 21988 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
16:19:40.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:40.539001 21988 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:19:40.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:40.541384 21988 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:19:40.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:40.542769 21988 hybrid_clock.cc:648] HybridClock initialized: now 1790180380542730 us; error 35 us; skew 500 ppm
16:19:40.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:40.543759 21988 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:19:40.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:40.550417 21988 webserver.cc:533] Webserver started at http://127.21.81.252:43905/ using document root <none> and password file <none>
16:19:40.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:40.551543 21988 fs_manager.cc:362] Metadata directory not provided
16:19:40.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:40.551813 21988 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:19:40.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:40.552388 21988 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
16:19:40.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:40.557103 21988 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster1397566977689697145/master-2/data/instance:
16:19:40.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "928a5fc61cb843498c3d9cfead175196"
16:19:40.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-23 16:19:40 on dist-test-slave-fqp8"
16:19:40.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:40.558498 21988 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster1397566977689697145/master-2/wal/instance:
16:19:40.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "928a5fc61cb843498c3d9cfead175196"
16:19:40.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-23 16:19:40 on dist-test-slave-fqp8"
16:19:40.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:40.566480 21988 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.006s sys 0.002s
16:19:40.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:40.572240 22015 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:19:40.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:40.573264 21988 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.005s sys 0.001s
16:19:40.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:40.573609 21988 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster1397566977689697145/master-2/data,/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster1397566977689697145/master-2/wal
16:19:40.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "928a5fc61cb843498c3d9cfead175196"
16:19:40.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-23 16:19:40 on dist-test-slave-fqp8"
16:19:40.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:40.574036 21988 fs_report.cc:389] FS layout report
16:19:40.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:19:40.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster1397566977689697145/master-2/wal
16:19:40.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster1397566977689697145/master-2/wal
16:19:40.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster1397566977689697145/master-2/data/data
16:19:40.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:19:40.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:19:40.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:19:40.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:19:40.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:19:40.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:19:40.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:19:40.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:19:40.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:19:40.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:19:40.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:19:40.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:40.633491 21988 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
16:19:40.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:40.635105 21988 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:19:40.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:40.635612 21988 kserver.cc:163] Server-wide thread pool size limit: 3276
16:19:40.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:40.704663 21988 rpc_server.cc:307] RPC server started. Bound to: 127.21.81.252:37327
16:19:40.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:40.704737 22066 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.21.81.252:37327 every 8 connection(s)
16:19:40.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:40.707739 21988 server_base.cc:1193] Dumped server information to /tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster1397566977689697145/master-2/data/info.pb
16:19:40.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:40.713287 21831 external_mini_cluster.cc:1434] Started /tmp/dist-test-tasko3a_Hu/build/tsan/bin/kudu as pid 21988
16:19:40.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:40.713781 21831 external_mini_cluster.cc:1448] Reading /tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster1397566977689697145/master-2/wal/instance
16:19:40.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:40.713863 22067 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
16:19:40.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:40.729921 22067 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.21.81.254" port: 40179 } has no permanent_uuid. Determining permanent_uuid...
16:19:40.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:40.757956 22067 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.21.81.253" port: 41791 } has no permanent_uuid. Determining permanent_uuid...
16:19:40.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:40.768908 22067 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.21.81.252" port: 37327 } has no permanent_uuid. Determining permanent_uuid...
16:19:40.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:40.786764 22067 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 928a5fc61cb843498c3d9cfead175196: Bootstrap starting.
16:19:40.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:40.792896 22067 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 928a5fc61cb843498c3d9cfead175196: Neither blocks nor log segments found. Creating new log.
16:19:40.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:40.794579 22067 log.cc:826] T 00000000000000000000000000000000 P 928a5fc61cb843498c3d9cfead175196: Log is configured to *not* fsync() on all Append() calls
16:19:40.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:40.799072 22067 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 928a5fc61cb843498c3d9cfead175196: No bootstrap required, opened a new log
16:19:40.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:40.811163 21986 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.81.252" port: 37327 } attempt: 7
16:19:40.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:40.820010 22067 raft_consensus.cc:359] T 00000000000000000000000000000000 P 928a5fc61cb843498c3d9cfead175196 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1b2d44848607499bb0e56d37b3328000" member_type: VOTER last_known_addr { host: "127.21.81.254" port: 40179 } } peers { permanent_uuid: "04b233e6e07044f2a4fbf857e0e0c4f8" member_type: VOTER last_known_addr { host: "127.21.81.253" port: 41791 } } peers { permanent_uuid: "928a5fc61cb843498c3d9cfead175196" member_type: VOTER last_known_addr { host: "127.21.81.252" port: 37327 } }
16:19:40.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:40.821370 22067 raft_consensus.cc:385] T 00000000000000000000000000000000 P 928a5fc61cb843498c3d9cfead175196 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
16:19:40.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:40.822001 22067 raft_consensus.cc:740] T 00000000000000000000000000000000 P 928a5fc61cb843498c3d9cfead175196 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 928a5fc61cb843498c3d9cfead175196, State: Initialized, Role: FOLLOWER
16:19:40.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:40.823195 22067 consensus_queue.cc:260] T 00000000000000000000000000000000 P 928a5fc61cb843498c3d9cfead175196 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1b2d44848607499bb0e56d37b3328000" member_type: VOTER last_known_addr { host: "127.21.81.254" port: 40179 } } peers { permanent_uuid: "04b233e6e07044f2a4fbf857e0e0c4f8" member_type: VOTER last_known_addr { host: "127.21.81.253" port: 41791 } } peers { permanent_uuid: "928a5fc61cb843498c3d9cfead175196" member_type: VOTER last_known_addr { host: "127.21.81.252" port: 37327 } }
16:19:40.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:40.826246 22077 sys_catalog.cc:455] T 00000000000000000000000000000000 P 928a5fc61cb843498c3d9cfead175196 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1b2d44848607499bb0e56d37b3328000" member_type: VOTER last_known_addr { host: "127.21.81.254" port: 40179 } } peers { permanent_uuid: "04b233e6e07044f2a4fbf857e0e0c4f8" member_type: VOTER last_known_addr { host: "127.21.81.253" port: 41791 } } peers { permanent_uuid: "928a5fc61cb843498c3d9cfead175196" member_type: VOTER last_known_addr { host: "127.21.81.252" port: 37327 } } }
16:19:40.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:40.827057 22077 sys_catalog.cc:458] T 00000000000000000000000000000000 P 928a5fc61cb843498c3d9cfead175196 [sys.catalog]: This master's current role is: FOLLOWER
16:19:40.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:40.827585 22067 sys_catalog.cc:565] T 00000000000000000000000000000000 P 928a5fc61cb843498c3d9cfead175196 [sys.catalog]: configured and running, proceeding with master startup.
16:19:40.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:40.829999 21986 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 04b233e6e07044f2a4fbf857e0e0c4f8: Bootstrap starting.
16:19:40.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:40.836465 21986 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 04b233e6e07044f2a4fbf857e0e0c4f8: Neither blocks nor log segments found. Creating new log.
16:19:40.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:40.839087 21986 log.cc:826] T 00000000000000000000000000000000 P 04b233e6e07044f2a4fbf857e0e0c4f8: Log is configured to *not* fsync() on all Append() calls
16:19:40.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:40.843611 21986 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 04b233e6e07044f2a4fbf857e0e0c4f8: No bootstrap required, opened a new log
16:19:40.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:40.861486 22089 catalog_manager.cc:1607] T 00000000000000000000000000000000 P 928a5fc61cb843498c3d9cfead175196: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:19:40.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:40.862329 22089 catalog_manager.cc:896] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:19:40.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:40.867370 22077 raft_consensus.cc:493] T 00000000000000000000000000000000 P 928a5fc61cb843498c3d9cfead175196 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
16:19:40.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:40.867888 22077 raft_consensus.cc:515] T 00000000000000000000000000000000 P 928a5fc61cb843498c3d9cfead175196 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1b2d44848607499bb0e56d37b3328000" member_type: VOTER last_known_addr { host: "127.21.81.254" port: 40179 } } peers { permanent_uuid: "04b233e6e07044f2a4fbf857e0e0c4f8" member_type: VOTER last_known_addr { host: "127.21.81.253" port: 41791 } } peers { permanent_uuid: "928a5fc61cb843498c3d9cfead175196" member_type: VOTER last_known_addr { host: "127.21.81.252" port: 37327 } }
16:19:40.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:40.870428 21986 raft_consensus.cc:359] T 00000000000000000000000000000000 P 04b233e6e07044f2a4fbf857e0e0c4f8 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1b2d44848607499bb0e56d37b3328000" member_type: VOTER last_known_addr { host: "127.21.81.254" port: 40179 } } peers { permanent_uuid: "04b233e6e07044f2a4fbf857e0e0c4f8" member_type: VOTER last_known_addr { host: "127.21.81.253" port: 41791 } } peers { permanent_uuid: "928a5fc61cb843498c3d9cfead175196" member_type: VOTER last_known_addr { host: "127.21.81.252" port: 37327 } }
16:19:40.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:40.871253 21986 raft_consensus.cc:385] T 00000000000000000000000000000000 P 04b233e6e07044f2a4fbf857e0e0c4f8 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
16:19:40.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:40.871522 21986 raft_consensus.cc:740] T 00000000000000000000000000000000 P 04b233e6e07044f2a4fbf857e0e0c4f8 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 04b233e6e07044f2a4fbf857e0e0c4f8, State: Initialized, Role: FOLLOWER
16:19:40.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:40.872566 21986 consensus_queue.cc:260] T 00000000000000000000000000000000 P 04b233e6e07044f2a4fbf857e0e0c4f8 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1b2d44848607499bb0e56d37b3328000" member_type: VOTER last_known_addr { host: "127.21.81.254" port: 40179 } } peers { permanent_uuid: "04b233e6e07044f2a4fbf857e0e0c4f8" member_type: VOTER last_known_addr { host: "127.21.81.253" port: 41791 } } peers { permanent_uuid: "928a5fc61cb843498c3d9cfead175196" member_type: VOTER last_known_addr { host: "127.21.81.252" port: 37327 } }
16:19:40.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:40.870522 22077 leader_election.cc:290] T 00000000000000000000000000000000 P 928a5fc61cb843498c3d9cfead175196 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 1b2d44848607499bb0e56d37b3328000 (127.21.81.254:40179), 04b233e6e07044f2a4fbf857e0e0c4f8 (127.21.81.253:41791)
16:19:40.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:40.881618 21986 sys_catalog.cc:565] T 00000000000000000000000000000000 P 04b233e6e07044f2a4fbf857e0e0c4f8 [sys.catalog]: configured and running, proceeding with master startup.
16:19:40.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:40.881731 21961 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "928a5fc61cb843498c3d9cfead175196" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "04b233e6e07044f2a4fbf857e0e0c4f8" is_pre_election: true
16:19:40.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:40.882647 21961 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 04b233e6e07044f2a4fbf857e0e0c4f8 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 928a5fc61cb843498c3d9cfead175196 in term 0.
16:19:40.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:40.884940 22019 leader_election.cc:304] T 00000000000000000000000000000000 P 928a5fc61cb843498c3d9cfead175196 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 04b233e6e07044f2a4fbf857e0e0c4f8, 928a5fc61cb843498c3d9cfead175196; no voters:
16:19:40.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:40.885977 22077 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 928a5fc61cb843498c3d9cfead175196 [term 0 FOLLOWER]: Leader pre-election won for term 1
16:19:40.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:40.886337 22077 raft_consensus.cc:493] T 00000000000000000000000000000000 P 928a5fc61cb843498c3d9cfead175196 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
16:19:40.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:40.886658 22077 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 928a5fc61cb843498c3d9cfead175196 [term 0 FOLLOWER]: Advancing to term 1
16:19:40.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:40.890722 22021 tablet.cc:2433] T 00000000000000000000000000000000 P 928a5fc61cb843498c3d9cfead175196: Can't schedule compaction. Clean time has not been advanced past its initial value.
16:19:40.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:40.880582 22091 sys_catalog.cc:455] T 00000000000000000000000000000000 P 04b233e6e07044f2a4fbf857e0e0c4f8 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1b2d44848607499bb0e56d37b3328000" member_type: VOTER last_known_addr { host: "127.21.81.254" port: 40179 } } peers { permanent_uuid: "04b233e6e07044f2a4fbf857e0e0c4f8" member_type: VOTER last_known_addr { host: "127.21.81.253" port: 41791 } } peers { permanent_uuid: "928a5fc61cb843498c3d9cfead175196" member_type: VOTER last_known_addr { host: "127.21.81.252" port: 37327 } } }
16:19:40.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:40.856491 21988 master_runner.cc:187] Error getting master registration for 127.21.81.254:40179: OK, instance_id { permanent_uuid: "1b2d44848607499bb0e56d37b3328000" instance_seqno: 1790180376588160 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
16:19:40.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:40.903018 21988 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
16:19:40.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:40.899612 22077 raft_consensus.cc:515] T 00000000000000000000000000000000 P 928a5fc61cb843498c3d9cfead175196 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1b2d44848607499bb0e56d37b3328000" member_type: VOTER last_known_addr { host: "127.21.81.254" port: 40179 } } peers { permanent_uuid: "04b233e6e07044f2a4fbf857e0e0c4f8" member_type: VOTER last_known_addr { host: "127.21.81.253" port: 41791 } } peers { permanent_uuid: "928a5fc61cb843498c3d9cfead175196" member_type: VOTER last_known_addr { host: "127.21.81.252" port: 37327 } }
16:19:40.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:40.904636 22091 sys_catalog.cc:458] T 00000000000000000000000000000000 P 04b233e6e07044f2a4fbf857e0e0c4f8 [sys.catalog]: This master's current role is: FOLLOWER
16:19:40.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:40.910413 22077 leader_election.cc:290] T 00000000000000000000000000000000 P 928a5fc61cb843498c3d9cfead175196 [CANDIDATE]: Term 1 election: Requested vote from peers 1b2d44848607499bb0e56d37b3328000 (127.21.81.254:40179), 04b233e6e07044f2a4fbf857e0e0c4f8 (127.21.81.253:41791)
16:19:40.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:40.910583 21961 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "928a5fc61cb843498c3d9cfead175196" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "04b233e6e07044f2a4fbf857e0e0c4f8"
16:19:40.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:40.911154 21961 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 04b233e6e07044f2a4fbf857e0e0c4f8 [term 0 FOLLOWER]: Advancing to term 1
16:19:40.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:40.919284 21961 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 04b233e6e07044f2a4fbf857e0e0c4f8 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 928a5fc61cb843498c3d9cfead175196 in term 1.
16:19:40.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:40.922433 22019 leader_election.cc:304] T 00000000000000000000000000000000 P 928a5fc61cb843498c3d9cfead175196 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 04b233e6e07044f2a4fbf857e0e0c4f8, 928a5fc61cb843498c3d9cfead175196; no voters:
16:19:40.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:40.923467 22077 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 928a5fc61cb843498c3d9cfead175196 [term 1 FOLLOWER]: Leader election won for term 1
16:19:40.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:40.926177 22077 raft_consensus.cc:697] T 00000000000000000000000000000000 P 928a5fc61cb843498c3d9cfead175196 [term 1 LEADER]: Becoming Leader. State: Replica: 928a5fc61cb843498c3d9cfead175196, State: Running, Role: LEADER
16:19:40.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:40.927431 22077 consensus_queue.cc:237] T 00000000000000000000000000000000 P 928a5fc61cb843498c3d9cfead175196 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1b2d44848607499bb0e56d37b3328000" member_type: VOTER last_known_addr { host: "127.21.81.254" port: 40179 } } peers { permanent_uuid: "04b233e6e07044f2a4fbf857e0e0c4f8" member_type: VOTER last_known_addr { host: "127.21.81.253" port: 41791 } } peers { permanent_uuid: "928a5fc61cb843498c3d9cfead175196" member_type: VOTER last_known_addr { host: "127.21.81.252" port: 37327 } }
16:19:40.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:40.939216 22100 sys_catalog.cc:455] T 00000000000000000000000000000000 P 928a5fc61cb843498c3d9cfead175196 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 928a5fc61cb843498c3d9cfead175196. Latest consensus state: current_term: 1 leader_uuid: "928a5fc61cb843498c3d9cfead175196" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1b2d44848607499bb0e56d37b3328000" member_type: VOTER last_known_addr { host: "127.21.81.254" port: 40179 } } peers { permanent_uuid: "04b233e6e07044f2a4fbf857e0e0c4f8" member_type: VOTER last_known_addr { host: "127.21.81.253" port: 41791 } } peers { permanent_uuid: "928a5fc61cb843498c3d9cfead175196" member_type: VOTER last_known_addr { host: "127.21.81.252" port: 37327 } } }
16:19:40.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:40.940878 22100 sys_catalog.cc:458] T 00000000000000000000000000000000 P 928a5fc61cb843498c3d9cfead175196 [sys.catalog]: This master's current role is: LEADER
16:19:40.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:40.967555 22105 catalog_manager.cc:1524] Loading table and tablet metadata into memory...
16:19:40.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:40.973923 22104 catalog_manager.cc:1607] T 00000000000000000000000000000000 P 04b233e6e07044f2a4fbf857e0e0c4f8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:19:40.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:40.974665 22104 catalog_manager.cc:896] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:19:40.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:40.975289 22105 catalog_manager.cc:1533] Initializing Kudu cluster ID...
16:19:40.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:40.947319 21913 master_runner.cc:187] Error getting master registration for 127.21.81.254:40179: OK, instance_id { permanent_uuid: "1b2d44848607499bb0e56d37b3328000" instance_seqno: 1790180376588160 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
16:19:40.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:40.989207 21913 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
16:19:41.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:41.001006 21961 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 04b233e6e07044f2a4fbf857e0e0c4f8 [term 1 FOLLOWER]: Refusing update from remote peer 928a5fc61cb843498c3d9cfead175196: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
16:19:41.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:41.006742 22100 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 928a5fc61cb843498c3d9cfead175196 [LEADER]: Connected to new peer: Peer: permanent_uuid: "04b233e6e07044f2a4fbf857e0e0c4f8" member_type: VOTER last_known_addr { host: "127.21.81.253" port: 41791 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
16:19:41.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:41.037889 22091 sys_catalog.cc:455] T 00000000000000000000000000000000 P 04b233e6e07044f2a4fbf857e0e0c4f8 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 928a5fc61cb843498c3d9cfead175196. Latest consensus state: current_term: 1 leader_uuid: "928a5fc61cb843498c3d9cfead175196" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1b2d44848607499bb0e56d37b3328000" member_type: VOTER last_known_addr { host: "127.21.81.254" port: 40179 } } peers { permanent_uuid: "04b233e6e07044f2a4fbf857e0e0c4f8" member_type: VOTER last_known_addr { host: "127.21.81.253" port: 41791 } } peers { permanent_uuid: "928a5fc61cb843498c3d9cfead175196" member_type: VOTER last_known_addr { host: "127.21.81.252" port: 37327 } } }
16:19:41.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:41.038745 22091 sys_catalog.cc:458] T 00000000000000000000000000000000 P 04b233e6e07044f2a4fbf857e0e0c4f8 [sys.catalog]: This master's current role is: FOLLOWER
16:19:41.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:41.044576 22107 mvcc.cc:204] Tried to move back new op lower bound from 7332578840549400576 to 7332578840294715392. Current Snapshot: MvccSnapshot[applied={T|T < 7332578840549400576}]
16:19:41.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:41.055167 22091 sys_catalog.cc:455] T 00000000000000000000000000000000 P 04b233e6e07044f2a4fbf857e0e0c4f8 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "928a5fc61cb843498c3d9cfead175196" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1b2d44848607499bb0e56d37b3328000" member_type: VOTER last_known_addr { host: "127.21.81.254" port: 40179 } } peers { permanent_uuid: "04b233e6e07044f2a4fbf857e0e0c4f8" member_type: VOTER last_known_addr { host: "127.21.81.253" port: 41791 } } peers { permanent_uuid: "928a5fc61cb843498c3d9cfead175196" member_type: VOTER last_known_addr { host: "127.21.81.252" port: 37327 } } }
16:19:41.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:41.055888 22091 sys_catalog.cc:458] T 00000000000000000000000000000000 P 04b233e6e07044f2a4fbf857e0e0c4f8 [sys.catalog]: This master's current role is: FOLLOWER
16:19:41.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:41.063274 22105 catalog_manager.cc:1396] Generated new cluster ID: d0757d7ed06b42a9a3e08129c1c8e84b
16:19:41.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:41.063597 22105 catalog_manager.cc:1544] Initializing Kudu internal certificate authority...
16:19:41.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:41.064230 22077 sys_catalog.cc:455] T 00000000000000000000000000000000 P 928a5fc61cb843498c3d9cfead175196 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "928a5fc61cb843498c3d9cfead175196" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1b2d44848607499bb0e56d37b3328000" member_type: VOTER last_known_addr { host: "127.21.81.254" port: 40179 } } peers { permanent_uuid: "04b233e6e07044f2a4fbf857e0e0c4f8" member_type: VOTER last_known_addr { host: "127.21.81.253" port: 41791 } } peers { permanent_uuid: "928a5fc61cb843498c3d9cfead175196" member_type: VOTER last_known_addr { host: "127.21.81.252" port: 37327 } } }
16:19:41.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:41.064926 22077 sys_catalog.cc:458] T 00000000000000000000000000000000 P 928a5fc61cb843498c3d9cfead175196 [sys.catalog]: This master's current role is: LEADER
16:19:41.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:41.108217 22105 catalog_manager.cc:1419] Generated new certificate authority record
16:19:41.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:41.111202 22105 catalog_manager.cc:1553] Loading token signing keys...
16:19:41.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:41.140699 22105 catalog_manager.cc:6219] T 00000000000000000000000000000000 P 928a5fc61cb843498c3d9cfead175196: Generated new TSK 0
16:19:41.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:41.142320 22105 catalog_manager.cc:1563] Initializing in-progress tserver states...
16:19:41.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:41.180720 21831 external_mini_cluster.cc:1372] Running /tmp/dist-test-tasko3a_Hu/build/tsan/bin/kudu
16:19:41.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasko3a_Hu/build/tsan/bin/kudu
16:19:41.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster1397566977689697145/ts-0/wal
16:19:41.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster1397566977689697145/ts-0/data
16:19:41.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:19:41.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:19:41.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:19:41.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:19:41.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:19:41.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:19:41.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster1397566977689697145/ts-0/logs
16:19:41.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster1397566977689697145/ts-0/data/info.pb
16:19:41.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:19:41.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:19:41.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:19:41.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:19:41.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:19:41.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:19:41.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:19:41.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:19:41.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
16:19:41.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:19:41.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.81.193:0
16:19:41.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.21.81.193
16:19:41.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.81.193
16:19:41.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:19:41.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.21.81.254:40179,127.21.81.253:41791,127.21.81.252:37327
16:19:41.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.81.212:37463
16:19:41.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:19:41.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:19:41.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
16:19:41.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:41.377059 21911 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.81.252" port: 37327 } attempt: 7
16:19:41.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:41.457259 21911 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1b2d44848607499bb0e56d37b3328000: Bootstrap starting.
16:19:41.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:41.478209 21911 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 1b2d44848607499bb0e56d37b3328000: Neither blocks nor log segments found. Creating new log.
16:19:41.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:41.485133 21911 log.cc:826] T 00000000000000000000000000000000 P 1b2d44848607499bb0e56d37b3328000: Log is configured to *not* fsync() on all Append() calls
16:19:41.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:41.518198 21911 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1b2d44848607499bb0e56d37b3328000: No bootstrap required, opened a new log
16:19:41.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:41.658679 21911 raft_consensus.cc:359] T 00000000000000000000000000000000 P 1b2d44848607499bb0e56d37b3328000 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1b2d44848607499bb0e56d37b3328000" member_type: VOTER last_known_addr { host: "127.21.81.254" port: 40179 } } peers { permanent_uuid: "04b233e6e07044f2a4fbf857e0e0c4f8" member_type: VOTER last_known_addr { host: "127.21.81.253" port: 41791 } } peers { permanent_uuid: "928a5fc61cb843498c3d9cfead175196" member_type: VOTER last_known_addr { host: "127.21.81.252" port: 37327 } }
16:19:41.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:41.662043 21911 raft_consensus.cc:385] T 00000000000000000000000000000000 P 1b2d44848607499bb0e56d37b3328000 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
16:19:41.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:41.663064 21911 raft_consensus.cc:740] T 00000000000000000000000000000000 P 1b2d44848607499bb0e56d37b3328000 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1b2d44848607499bb0e56d37b3328000, State: Initialized, Role: FOLLOWER
16:19:41.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:41.681654 21911 consensus_queue.cc:260] T 00000000000000000000000000000000 P 1b2d44848607499bb0e56d37b3328000 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1b2d44848607499bb0e56d37b3328000" member_type: VOTER last_known_addr { host: "127.21.81.254" port: 40179 } } peers { permanent_uuid: "04b233e6e07044f2a4fbf857e0e0c4f8" member_type: VOTER last_known_addr { host: "127.21.81.253" port: 41791 } } peers { permanent_uuid: "928a5fc61cb843498c3d9cfead175196" member_type: VOTER last_known_addr { host: "127.21.81.252" port: 37327 } }
16:19:41.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:41.698801 22116 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1b2d44848607499bb0e56d37b3328000 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1b2d44848607499bb0e56d37b3328000" member_type: VOTER last_known_addr { host: "127.21.81.254" port: 40179 } } peers { permanent_uuid: "04b233e6e07044f2a4fbf857e0e0c4f8" member_type: VOTER last_known_addr { host: "127.21.81.253" port: 41791 } } peers { permanent_uuid: "928a5fc61cb843498c3d9cfead175196" member_type: VOTER last_known_addr { host: "127.21.81.252" port: 37327 } } }
16:19:41.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:41.699582 22116 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1b2d44848607499bb0e56d37b3328000 [sys.catalog]: This master's current role is: FOLLOWER
16:19:41.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:41.706621 21886 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "928a5fc61cb843498c3d9cfead175196" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1b2d44848607499bb0e56d37b3328000" is_pre_election: true
16:19:41.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:41.707470 21886 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 1b2d44848607499bb0e56d37b3328000 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 928a5fc61cb843498c3d9cfead175196 in term 0.
16:19:41.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:41.708911 21884 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 1b2d44848607499bb0e56d37b3328000 [term 0 FOLLOWER]: Advancing to term 1
16:19:41.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:41.714005 21885 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "928a5fc61cb843498c3d9cfead175196" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1b2d44848607499bb0e56d37b3328000"
16:19:41.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:41.720358 21884 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 1b2d44848607499bb0e56d37b3328000 [term 1 FOLLOWER]: Refusing update from remote peer 928a5fc61cb843498c3d9cfead175196: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
16:19:41.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:41.729947 22117 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 928a5fc61cb843498c3d9cfead175196 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1b2d44848607499bb0e56d37b3328000" member_type: VOTER last_known_addr { host: "127.21.81.254" port: 40179 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
16:19:41.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:41.729322 21911 sys_catalog.cc:565] T 00000000000000000000000000000000 P 1b2d44848607499bb0e56d37b3328000 [sys.catalog]: configured and running, proceeding with master startup.
16:19:41.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:41.819386 22116 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1b2d44848607499bb0e56d37b3328000 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 928a5fc61cb843498c3d9cfead175196. Latest consensus state: current_term: 1 leader_uuid: "928a5fc61cb843498c3d9cfead175196" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1b2d44848607499bb0e56d37b3328000" member_type: VOTER last_known_addr { host: "127.21.81.254" port: 40179 } } peers { permanent_uuid: "04b233e6e07044f2a4fbf857e0e0c4f8" member_type: VOTER last_known_addr { host: "127.21.81.253" port: 41791 } } peers { permanent_uuid: "928a5fc61cb843498c3d9cfead175196" member_type: VOTER last_known_addr { host: "127.21.81.252" port: 37327 } } }
16:19:41.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:41.819571 22124 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1b2d44848607499bb0e56d37b3328000 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "928a5fc61cb843498c3d9cfead175196" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1b2d44848607499bb0e56d37b3328000" member_type: VOTER last_known_addr { host: "127.21.81.254" port: 40179 } } peers { permanent_uuid: "04b233e6e07044f2a4fbf857e0e0c4f8" member_type: VOTER last_known_addr { host: "127.21.81.253" port: 41791 } } peers { permanent_uuid: "928a5fc61cb843498c3d9cfead175196" member_type: VOTER last_known_addr { host: "127.21.81.252" port: 37327 } } }
16:19:41.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:41.820096 22116 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1b2d44848607499bb0e56d37b3328000 [sys.catalog]: This master's current role is: FOLLOWER
16:19:41.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:41.820324 22124 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1b2d44848607499bb0e56d37b3328000 [sys.catalog]: This master's current role is: FOLLOWER
16:19:41.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:41.886646 22137 catalog_manager.cc:1308] Loaded cluster ID: d0757d7ed06b42a9a3e08129c1c8e84b
16:19:41.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:41.886924 22137 catalog_manager.cc:1601] T 00000000000000000000000000000000 P 1b2d44848607499bb0e56d37b3328000: loading cluster ID for follower catalog manager: success
16:19:41.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:41.905093 22137 catalog_manager.cc:1623] T 00000000000000000000000000000000 P 1b2d44848607499bb0e56d37b3328000: acquiring CA information for follower catalog manager: success
16:19:41.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:41.918465 22137 catalog_manager.cc:1651] T 00000000000000000000000000000000 P 1b2d44848607499bb0e56d37b3328000: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
16:19:41.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:41.940305 22112 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
16:19:41.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:41.941390 22112 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:19:41.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:41.942435 22112 flags.cc:432] Enabled unsafe flag: --never_fsync=true
16:19:41.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:41.987509 22104 catalog_manager.cc:1308] Loaded cluster ID: d0757d7ed06b42a9a3e08129c1c8e84b
16:19:41.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:41.988234 22104 catalog_manager.cc:1601] T 00000000000000000000000000000000 P 04b233e6e07044f2a4fbf857e0e0c4f8: loading cluster ID for follower catalog manager: success
16:19:41.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:41.995836 22104 catalog_manager.cc:1623] T 00000000000000000000000000000000 P 04b233e6e07044f2a4fbf857e0e0c4f8: acquiring CA information for follower catalog manager: success
16:19:42.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:42.004936 22112 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
16:19:42.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:42.006753 22112 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.21.81.193
16:19:42.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:42.011423 22104 catalog_manager.cc:1651] T 00000000000000000000000000000000 P 04b233e6e07044f2a4fbf857e0e0c4f8: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
16:19:42.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:42.075728 22112 tablet_server_runner.cc:78] Tablet server non-default flags:
16:19:42.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster1397566977689697145/ts-0/logs
16:19:42.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:19:42.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:19:42.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:19:42.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.81.212:37463
16:19:42.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:19:42.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:19:42.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster1397566977689697145/ts-0/data
16:19:42.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster1397566977689697145/ts-0/wal
16:19:42.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:19:42.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:19:42.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.81.193:0
16:19:42.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:19:42.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:19:42.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:19:42.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster1397566977689697145/ts-0/data/info.pb
16:19:42.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.81.193
16:19:42.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:19:42.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.21.81.254:40179,127.21.81.253:41791,127.21.81.252:37327
16:19:42.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:19:42.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:19:42.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:19:42.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:19:42.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:19:42.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.21.81.193
16:19:42.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:19:42.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
16:19:42.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
16:19:42.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision faa920b8364ca43fd9d419253dab85c46533dc98
16:19:42.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:19:42.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 23 Sep 2026 15:47:26 UTC on e7f111948823
16:19:42.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 13962
16:19:42.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:19:42.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:42.087224 22112 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:19:42.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:42.089951 22112 file_cache.cc:504] Constructed file cache file cache with capacity 419430
16:19:42.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:42.117727 22141 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:19:42.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:42.350172 22117 sys_catalog.cc:455] T 00000000000000000000000000000000 P 928a5fc61cb843498c3d9cfead175196 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "928a5fc61cb843498c3d9cfead175196" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1b2d44848607499bb0e56d37b3328000" member_type: VOTER last_known_addr { host: "127.21.81.254" port: 40179 } } peers { permanent_uuid: "04b233e6e07044f2a4fbf857e0e0c4f8" member_type: VOTER last_known_addr { host: "127.21.81.253" port: 41791 } } peers { permanent_uuid: "928a5fc61cb843498c3d9cfead175196" member_type: VOTER last_known_addr { host: "127.21.81.252" port: 37327 } } }
16:19:42.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:42.352442 22117 sys_catalog.cc:458] T 00000000000000000000000000000000 P 928a5fc61cb843498c3d9cfead175196 [sys.catalog]: This master's current role is: LEADER
16:19:43.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:42.129377 22142 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:19:43.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:43.968804 22145 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:19:43.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:43.970950 22143 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1841 milliseconds
16:19:43.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:43.971032 22112 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
16:19:43.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:43.972497 22112 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:19:43.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:43.983669 22112 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:19:43.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:43.985351 22112 hybrid_clock.cc:648] HybridClock initialized: now 1790180383985304 us; error 52 us; skew 500 ppm
16:19:43.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:43.986685 22112 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:19:44.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:44.005692 22112 webserver.cc:533] Webserver started at http://127.21.81.193:41385/ using document root <none> and password file <none>
16:19:44.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:44.007364 22112 fs_manager.cc:362] Metadata directory not provided
16:19:44.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:44.007905 22112 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:19:44.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:44.008693 22112 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
16:19:44.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:44.015172 22112 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster1397566977689697145/ts-0/data/instance:
16:19:44.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cd61838f498a469ead5213a1db325991"
16:19:44.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-23 16:19:44 on dist-test-slave-fqp8"
16:19:44.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:44.016997 22112 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster1397566977689697145/ts-0/wal/instance:
16:19:44.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cd61838f498a469ead5213a1db325991"
16:19:44.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-23 16:19:44 on dist-test-slave-fqp8"
16:19:44.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:44.033506 22112 fs_manager.cc:696] Time spent creating directory manager: real 0.015s user 0.007s sys 0.006s
16:19:44.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:44.046959 22158 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:19:44.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:44.048657 22112 fs_manager.cc:730] Time spent opening block manager: real 0.010s user 0.001s sys 0.007s
16:19:44.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:44.049284 22112 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster1397566977689697145/ts-0/data,/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster1397566977689697145/ts-0/wal
16:19:44.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cd61838f498a469ead5213a1db325991"
16:19:44.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-23 16:19:44 on dist-test-slave-fqp8"
16:19:44.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:44.050117 22112 fs_report.cc:389] FS layout report
16:19:44.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:19:44.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster1397566977689697145/ts-0/wal
16:19:44.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster1397566977689697145/ts-0/wal
16:19:44.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster1397566977689697145/ts-0/data/data
16:19:44.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:19:44.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:19:44.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:19:44.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:19:44.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:19:44.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:19:44.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:19:44.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:19:44.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:19:44.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:19:44.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:19:44.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:44.157358 22112 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
16:19:44.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:44.159832 22112 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:19:44.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:44.160730 22112 kserver.cc:163] Server-wide thread pool size limit: 3276
16:19:44.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:44.173435 22112 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
16:19:44.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:44.185572 22112 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
16:19:44.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:44.186223 22112 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.001s user 0.000s sys 0.001s
16:19:44.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:44.186764 22112 ts_tablet_manager.cc:616] Registered 0 tablets
16:19:44.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:44.187229 22112 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
16:19:44.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:44.499260 22112 rpc_server.cc:307] RPC server started. Bound to: 127.21.81.193:34949
16:19:44.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:44.499418 22273 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.21.81.193:34949 every 8 connection(s)
16:19:44.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:44.504048 22112 server_base.cc:1193] Dumped server information to /tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster1397566977689697145/ts-0/data/info.pb
16:19:44.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:44.506026 21831 external_mini_cluster.cc:1434] Started /tmp/dist-test-tasko3a_Hu/build/tsan/bin/kudu as pid 22112
16:19:44.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:44.506456 21831 external_mini_cluster.cc:1448] Reading /tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster1397566977689697145/ts-0/wal/instance
16:19:44.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:44.524143 21831 external_mini_cluster.cc:1372] Running /tmp/dist-test-tasko3a_Hu/build/tsan/bin/kudu
16:19:44.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasko3a_Hu/build/tsan/bin/kudu
16:19:44.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster1397566977689697145/ts-1/wal
16:19:44.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster1397566977689697145/ts-1/data
16:19:44.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:19:44.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:19:44.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:19:44.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:19:44.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:19:44.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:19:44.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster1397566977689697145/ts-1/logs
16:19:44.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster1397566977689697145/ts-1/data/info.pb
16:19:44.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:19:44.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:19:44.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:19:44.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:19:44.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:19:44.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:19:44.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:19:44.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:19:44.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
16:19:44.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:19:44.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.81.194:0
16:19:44.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.21.81.194
16:19:44.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.81.194
16:19:44.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:19:44.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.21.81.254:40179,127.21.81.253:41791,127.21.81.252:37327
16:19:44.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.81.212:37463
16:19:44.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:19:44.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:19:44.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
16:19:44.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:44.639479 22275 heartbeater.cc:344] Connected to a master server at 127.21.81.252:37327
16:19:44.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:44.639935 22275 heartbeater.cc:461] Registering TS with master...
16:19:44.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:44.641243 22275 heartbeater.cc:507] Master 127.21.81.252:37327 requested a full tablet report, sending...
16:19:44.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:44.645496 22032 ts_manager.cc:194] Registered new tserver with Master: cd61838f498a469ead5213a1db325991 (127.21.81.193:34949)
16:19:44.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:44.651721 22032 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.21.81.193:44585
16:19:44.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:44.656252 22274 heartbeater.cc:344] Connected to a master server at 127.21.81.253:41791
16:19:44.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:44.656606 22274 heartbeater.cc:461] Registering TS with master...
16:19:44.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:44.657272 22274 heartbeater.cc:507] Master 127.21.81.253:41791 requested a full tablet report, sending...
16:19:44.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:44.660950 21951 ts_manager.cc:194] Registered new tserver with Master: cd61838f498a469ead5213a1db325991 (127.21.81.193:34949)
16:19:44.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:44.664681 22279 heartbeater.cc:344] Connected to a master server at 127.21.81.254:40179
16:19:44.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:44.665223 22279 heartbeater.cc:461] Registering TS with master...
16:19:44.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:44.666023 22279 heartbeater.cc:507] Master 127.21.81.254:40179 requested a full tablet report, sending...
16:19:44.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:44.669194 21876 ts_manager.cc:194] Registered new tserver with Master: cd61838f498a469ead5213a1db325991 (127.21.81.193:34949)
16:19:45.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:45.002457 22278 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
16:19:45.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:45.002923 22278 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:19:45.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:45.003314 22278 flags.cc:432] Enabled unsafe flag: --never_fsync=true
16:19:45.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:45.038987 22278 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
16:19:45.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:45.040108 22278 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.21.81.194
16:19:45.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:45.077453 22278 tablet_server_runner.cc:78] Tablet server non-default flags:
16:19:45.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster1397566977689697145/ts-1/logs
16:19:45.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:19:45.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:19:45.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:19:45.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.81.212:37463
16:19:45.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:19:45.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:19:45.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster1397566977689697145/ts-1/data
16:19:45.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster1397566977689697145/ts-1/wal
16:19:45.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:19:45.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:19:45.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.81.194:0
16:19:45.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:19:45.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:19:45.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:19:45.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster1397566977689697145/ts-1/data/info.pb
16:19:45.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.81.194
16:19:45.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:19:45.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.21.81.254:40179,127.21.81.253:41791,127.21.81.252:37327
16:19:45.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:19:45.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:19:45.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:19:45.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:19:45.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:19:45.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.21.81.194
16:19:45.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:19:45.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
16:19:45.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
16:19:45.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision faa920b8364ca43fd9d419253dab85c46533dc98
16:19:45.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:19:45.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 23 Sep 2026 15:47:26 UTC on e7f111948823
16:19:45.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 13962
16:19:45.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:19:45.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:45.078799 22278 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:19:45.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:45.080338 22278 file_cache.cc:504] Constructed file cache file cache with capacity 419430
16:19:45.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:45.095846 22291 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:19:45.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:45.645066 21907 debug-util.cc:398] Leaking SignalData structure 0x7b08000a4b00 after lost signal to thread 21844
16:19:45.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:45.646525 21907 debug-util.cc:398] Leaking SignalData structure 0x7b08000a9aa0 after lost signal to thread 21910
16:19:45.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:45.657104 22275 heartbeater.cc:499] Master 127.21.81.252:37327 was elected leader, sending a full tablet report...
16:19:46.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:45.100638 22292 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:19:46.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:45.101657 22278 server_base.cc:1061] running on GCE node
16:19:46.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:45.101034 22294 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:19:46.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:46.335100 22278 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:19:46.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:46.337940 22278 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:19:46.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:46.339610 22278 hybrid_clock.cc:648] HybridClock initialized: now 1790180386339539 us; error 87 us; skew 500 ppm
16:19:46.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:46.340492 22278 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:19:46.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:46.347944 22278 webserver.cc:533] Webserver started at http://127.21.81.194:39531/ using document root <none> and password file <none>
16:19:46.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:46.348922 22278 fs_manager.cc:362] Metadata directory not provided
16:19:46.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:46.349164 22278 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:19:46.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:46.349745 22278 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
16:19:46.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:46.354274 22278 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster1397566977689697145/ts-1/data/instance:
16:19:46.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "773b2745aa3a4371a9af135d4056c08c"
16:19:46.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-23 16:19:46 on dist-test-slave-fqp8"
16:19:46.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:46.355294 22278 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster1397566977689697145/ts-1/wal/instance:
16:19:46.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "773b2745aa3a4371a9af135d4056c08c"
16:19:46.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-23 16:19:46 on dist-test-slave-fqp8"
16:19:46.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:46.362768 22278 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.008s sys 0.000s
16:19:46.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:46.368832 22302 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:19:46.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:46.370072 22278 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.006s sys 0.000s
16:19:46.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:46.370342 22278 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster1397566977689697145/ts-1/data,/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster1397566977689697145/ts-1/wal
16:19:46.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "773b2745aa3a4371a9af135d4056c08c"
16:19:46.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-23 16:19:46 on dist-test-slave-fqp8"
16:19:46.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:46.370638 22278 fs_report.cc:389] FS layout report
16:19:46.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:19:46.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster1397566977689697145/ts-1/wal
16:19:46.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster1397566977689697145/ts-1/wal
16:19:46.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster1397566977689697145/ts-1/data/data
16:19:46.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:19:46.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:19:46.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:19:46.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:19:46.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:19:46.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:19:46.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:19:46.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:19:46.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:19:46.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:19:46.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:19:46.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:46.423811 22278 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
16:19:46.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:46.425441 22278 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:19:46.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:46.425992 22278 kserver.cc:163] Server-wide thread pool size limit: 3276
16:19:46.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:46.429378 22278 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
16:19:46.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:46.434638 22278 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
16:19:46.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:46.434865 22278 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
16:19:46.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:46.435182 22278 ts_tablet_manager.cc:616] Registered 0 tablets
16:19:46.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:46.435425 22278 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
16:19:46.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:46.615820 22278 rpc_server.cc:307] RPC server started. Bound to: 127.21.81.194:35931
16:19:46.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:46.615931 22414 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.21.81.194:35931 every 8 connection(s)
16:19:46.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:46.618644 22278 server_base.cc:1193] Dumped server information to /tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster1397566977689697145/ts-1/data/info.pb
16:19:46.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:46.627456 21831 external_mini_cluster.cc:1434] Started /tmp/dist-test-tasko3a_Hu/build/tsan/bin/kudu as pid 22278
16:19:46.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:46.628355 21831 external_mini_cluster.cc:1448] Reading /tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster1397566977689697145/ts-1/wal/instance
16:19:46.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:46.643195 21831 external_mini_cluster.cc:1372] Running /tmp/dist-test-tasko3a_Hu/build/tsan/bin/kudu
16:19:46.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasko3a_Hu/build/tsan/bin/kudu
16:19:46.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster1397566977689697145/ts-2/wal
16:19:46.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster1397566977689697145/ts-2/data
16:19:46.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:19:46.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:19:46.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:19:46.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:19:46.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:19:46.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:19:46.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster1397566977689697145/ts-2/logs
16:19:46.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster1397566977689697145/ts-2/data/info.pb
16:19:46.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:19:46.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:19:46.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:19:46.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:19:46.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:19:46.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:19:46.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:19:46.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:19:46.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
16:19:46.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:19:46.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.81.195:0
16:19:46.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.21.81.195
16:19:46.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.81.195
16:19:46.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:19:46.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.21.81.254:40179,127.21.81.253:41791,127.21.81.252:37327
16:19:46.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.81.212:37463
16:19:46.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:19:46.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:19:46.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
16:19:46.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:46.679482 22416 heartbeater.cc:344] Connected to a master server at 127.21.81.252:37327
16:19:46.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:46.680033 22416 heartbeater.cc:461] Registering TS with master...
16:19:46.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:46.680958 22417 heartbeater.cc:344] Connected to a master server at 127.21.81.254:40179
16:19:46.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:46.681321 22417 heartbeater.cc:461] Registering TS with master...
16:19:46.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:46.681396 22416 heartbeater.cc:507] Master 127.21.81.252:37327 requested a full tablet report, sending...
16:19:46.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:46.682282 22417 heartbeater.cc:507] Master 127.21.81.254:40179 requested a full tablet report, sending...
16:19:46.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:46.683070 22415 heartbeater.cc:344] Connected to a master server at 127.21.81.253:41791
16:19:46.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:46.683427 22415 heartbeater.cc:461] Registering TS with master...
16:19:46.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:46.684304 22415 heartbeater.cc:507] Master 127.21.81.253:41791 requested a full tablet report, sending...
16:19:46.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:46.685388 21876 ts_manager.cc:194] Registered new tserver with Master: 773b2745aa3a4371a9af135d4056c08c (127.21.81.194:35931)
16:19:46.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:46.687287 22032 ts_manager.cc:194] Registered new tserver with Master: 773b2745aa3a4371a9af135d4056c08c (127.21.81.194:35931)
16:19:46.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:46.687861 21951 ts_manager.cc:194] Registered new tserver with Master: 773b2745aa3a4371a9af135d4056c08c (127.21.81.194:35931)
16:19:46.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:46.689168 22032 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.21.81.194:35393
16:19:47.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:47.004226 22425 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
16:19:47.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:47.004773 22425 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:19:47.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:47.005347 22425 flags.cc:432] Enabled unsafe flag: --never_fsync=true
16:19:47.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:47.042217 22425 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
16:19:47.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:47.043284 22425 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.21.81.195
16:19:47.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:47.082540 22425 tablet_server_runner.cc:78] Tablet server non-default flags:
16:19:47.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster1397566977689697145/ts-2/logs
16:19:47.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:19:47.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:19:47.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:19:47.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.81.212:37463
16:19:47.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:19:47.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:19:47.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster1397566977689697145/ts-2/data
16:19:47.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster1397566977689697145/ts-2/wal
16:19:47.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:19:47.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:19:47.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.81.195:0
16:19:47.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:19:47.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:19:47.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:19:47.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster1397566977689697145/ts-2/data/info.pb
16:19:47.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.81.195
16:19:47.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:19:47.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.21.81.254:40179,127.21.81.253:41791,127.21.81.252:37327
16:19:47.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:19:47.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:19:47.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:19:47.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:19:47.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:19:47.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.21.81.195
16:19:47.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:19:47.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
16:19:47.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
16:19:47.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision faa920b8364ca43fd9d419253dab85c46533dc98
16:19:47.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:19:47.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 23 Sep 2026 15:47:26 UTC on e7f111948823
16:19:47.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 13962
16:19:47.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:19:47.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:47.084304 22425 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:19:47.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:47.086616 22425 file_cache.cc:504] Constructed file cache file cache with capacity 419430
16:19:47.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:47.100353 22432 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:19:47.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:47.692833 22416 heartbeater.cc:499] Master 127.21.81.252:37327 was elected leader, sending a full tablet report...
16:19:48.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:47.104813 22433 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:19:48.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:47.106140 22425 server_base.cc:1061] running on GCE node
16:19:48.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:47.105099 22435 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:19:48.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:48.309149 22425 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:19:48.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:48.311506 22425 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:19:48.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:48.312888 22425 hybrid_clock.cc:648] HybridClock initialized: now 1790180388312858 us; error 36 us; skew 500 ppm
16:19:48.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:48.313720 22425 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:19:48.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:48.319741 22425 webserver.cc:533] Webserver started at http://127.21.81.195:35669/ using document root <none> and password file <none>
16:19:48.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:48.320731 22425 fs_manager.cc:362] Metadata directory not provided
16:19:48.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:48.320971 22425 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:19:48.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:48.321486 22425 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
16:19:48.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:48.325716 22425 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster1397566977689697145/ts-2/data/instance:
16:19:48.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6c7b63bb641f416fab2c445ad117fe83"
16:19:48.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-23 16:19:48 on dist-test-slave-fqp8"
16:19:48.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:48.326924 22425 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster1397566977689697145/ts-2/wal/instance:
16:19:48.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6c7b63bb641f416fab2c445ad117fe83"
16:19:48.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-23 16:19:48 on dist-test-slave-fqp8"
16:19:48.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:48.334156 22425 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.007s sys 0.000s
16:19:48.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:48.339589 22442 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:19:48.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:48.340742 22425 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
16:19:48.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:48.341120 22425 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster1397566977689697145/ts-2/data,/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster1397566977689697145/ts-2/wal
16:19:48.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6c7b63bb641f416fab2c445ad117fe83"
16:19:48.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-23 16:19:48 on dist-test-slave-fqp8"
16:19:48.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:48.341411 22425 fs_report.cc:389] FS layout report
16:19:48.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:19:48.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster1397566977689697145/ts-2/wal
16:19:48.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster1397566977689697145/ts-2/wal
16:19:48.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster1397566977689697145/ts-2/data/data
16:19:48.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:19:48.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:19:48.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:19:48.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:19:48.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:19:48.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:19:48.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:19:48.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:19:48.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:19:48.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:19:48.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:19:48.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:48.390360 22425 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
16:19:48.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:48.391851 22425 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:19:48.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:48.392246 22425 kserver.cc:163] Server-wide thread pool size limit: 3276
16:19:48.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:48.394739 22425 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
16:19:48.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:48.399118 22425 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
16:19:48.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:48.399308 22425 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
16:19:48.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:48.399494 22425 ts_tablet_manager.cc:616] Registered 0 tablets
16:19:48.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:48.399684 22425 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
16:19:48.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:48.547359 22425 rpc_server.cc:307] RPC server started. Bound to: 127.21.81.195:42327
16:19:48.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:48.547487 22555 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.21.81.195:42327 every 8 connection(s)
16:19:48.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:48.549780 22425 server_base.cc:1193] Dumped server information to /tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster1397566977689697145/ts-2/data/info.pb
16:19:48.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:48.554895 21831 external_mini_cluster.cc:1434] Started /tmp/dist-test-tasko3a_Hu/build/tsan/bin/kudu as pid 22425
16:19:48.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:48.555438 21831 external_mini_cluster.cc:1448] Reading /tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster1397566977689697145/ts-2/wal/instance
16:19:48.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:48.586057 22556 heartbeater.cc:344] Connected to a master server at 127.21.81.253:41791
16:19:48.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:48.586558 22556 heartbeater.cc:461] Registering TS with master...
16:19:48.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:48.587860 22556 heartbeater.cc:507] Master 127.21.81.253:41791 requested a full tablet report, sending...
16:19:48.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:48.590926 21951 ts_manager.cc:194] Registered new tserver with Master: 6c7b63bb641f416fab2c445ad117fe83 (127.21.81.195:42327)
16:19:48.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:48.592533 22557 heartbeater.cc:344] Connected to a master server at 127.21.81.252:37327
16:19:48.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:48.592840 22557 heartbeater.cc:461] Registering TS with master...
16:19:48.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:48.593715 22557 heartbeater.cc:507] Master 127.21.81.252:37327 requested a full tablet report, sending...
16:19:48.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:48.595824 22032 ts_manager.cc:194] Registered new tserver with Master: 6c7b63bb641f416fab2c445ad117fe83 (127.21.81.195:42327)
16:19:48.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:48.596130 22558 heartbeater.cc:344] Connected to a master server at 127.21.81.254:40179
16:19:48.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:48.596464 22558 heartbeater.cc:461] Registering TS with master...
16:19:48.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:48.597328 22558 heartbeater.cc:507] Master 127.21.81.254:40179 requested a full tablet report, sending...
16:19:48.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:48.597519 22032 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.21.81.195:49717
16:19:48.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:48.600224 21876 ts_manager.cc:194] Registered new tserver with Master: 6c7b63bb641f416fab2c445ad117fe83 (127.21.81.195:42327)
16:19:48.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:48.607249 21831 external_mini_cluster.cc:955] 3 TS(s) registered with all masters
16:19:48.618 [DEBUG - main] (MiniKuduCluster.java:195) Response:
16:19:48.620 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
16:19:48.629 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.21.81.254"
port: 40179
}
bound_http_address {
host: "127.21.81.254"
port: 46695
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.21.81.253"
port: 41791
}
bound_http_address {
host: "127.21.81.253"
port: 46121
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.21.81.252"
port: 37327
}
bound_http_address {
host: "127.21.81.252"
port: 43905
}
}
}
16:19:48.632 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
16:19:48.635 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.21.81.193"
port: 34949
}
bound_http_address {
host: "127.21.81.193"
port: 41385
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.21.81.194"
port: 35931
}
bound_http_address {
host: "127.21.81.194"
port: 39531
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.21.81.195"
port: 42327
}
bound_http_address {
host: "127.21.81.195"
port: 35669
}
}
}
16:19:48.635 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
16:19:48.821 [DEBUG - main] (CompositeStrategy.java:59) Unable to load modern Subject methods, relying only on legacy methods
java.lang.NoSuchMethodException: javax.security.auth.Subject.current()
at java.base/java.lang.Class.getDeclaredMethod(Class.java:2677)
at org.apache.kudu.client.internals.ModernStrategy.<init>(ModernStrategy.java:51)
at org.apache.kudu.client.internals.CompositeStrategy.<init>(CompositeStrategy.java:52)
at org.apache.kudu.client.internals.CompositeStrategy.<clinit>(CompositeStrategy.java:41)
at org.apache.kudu.client.internals.SecurityManagerCompatibility.get(SecurityManagerCompatibility.java:43)
at org.apache.kudu.client.SecurityContext.setupSubject(SecurityContext.java:155)
at org.apache.kudu.client.SecurityContext.<init>(SecurityContext.java:138)
at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:431)
at org.apache.kudu.client.AsyncKuduClient$AsyncKuduClientBuilder.build(AsyncKuduClient.java:3109)
at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:161)
at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runners.Suite.runChild(Suite.java:128)
at org.junit.runners.Suite.runChild(Suite.java:27)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runner.JUnitCore.run(JUnitCore.java:137)
at org.junit.runner.JUnitCore.run(JUnitCore.java:115)
at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77)
at org.junit.runner.JUnitCore.main(JUnitCore.java:36)
java.lang.NoSuchMethodException: javax.security.auth.Subject.current()
at java.base/java.lang.Class.getDeclaredMethod(Class.java:2677)
at org.apache.kudu.client.internals.ModernStrategy.<init>(ModernStrategy.java:51)
at org.apache.kudu.client.internals.CompositeStrategy.<init>(CompositeStrategy.java:52)
at org.apache.kudu.client.internals.CompositeStrategy.<clinit>(CompositeStrategy.java:41)
at org.apache.kudu.client.internals.SecurityManagerCompatibility.get(SecurityManagerCompatibility.java:43)
at org.apache.kudu.client.SecurityContext.setupSubject(SecurityContext.java:155)
at org.apache.kudu.client.SecurityContext.<init>(SecurityContext.java:138)
at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:431)
at org.apache.kudu.client.AsyncKuduClient$AsyncKuduClientBuilder.build(AsyncKuduClient.java:3109)
at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:161)
at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runners.Suite.runChild(Suite.java:128)
at org.junit.runners.Suite.runChild(Suite.java:27)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runner.JUnitCore.run(JUnitCore.java:137)
at org.junit.runner.JUnitCore.run(JUnitCore.java:115)
at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77)
at org.junit.runner.JUnitCore.main(JUnitCore.java:36)
16:19:48.877 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
16:19:48.962 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-tasko3a_Hu/test-tmp
16:19:48.963 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
16:19:48.963 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-tasko3a_Hu/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
16:19:48.966 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: true
cluster_root: "/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834"
extra_master_flags: "--webserver_require_spnego=false"
extra_master_flags: "--authn_token_validity_seconds=10"
extra_tserver_flags: "--webserver_require_spnego=false"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
16:19:49.293 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
16:19:49.294 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
16:19:49.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:49.298352 22568 mini_oidc.cc:150] Starting JWKS server
16:19:49.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:49.301379 22568 webserver.cc:533] Webserver started at http://127.0.0.1:41371/ using document root <none> and password file <none>
16:19:49.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:49.301829 22568 mini_oidc.cc:186] Starting OIDC Discovery server
16:19:49.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:49.303417 22568 webserver.cc:533] Webserver started at http://127.0.0.1:33879/ using document root <none> and password file <none>
16:19:49.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
16:19:49.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/krb5kdc/principal' for realm 'KRBTEST.COM',
16:19:49.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
16:19:49.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 23 16:19:49 dist-test-slave-fqp8 krb5kdc[22580](info): setting up network...
16:19:49.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
16:19:49.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 23 16:19:49 dist-test-slave-fqp8 krb5kdc[22580](info): set up 2 sockets
16:19:49.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 23 16:19:49 dist-test-slave-fqp8 krb5kdc[22580](info): commencing operation
16:19:49.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
16:19:49.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:49.601117 22557 heartbeater.cc:499] Master 127.21.81.252:37327 was elected leader, sending a full tablet report...
16:19:51.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:51.409266 22568 mini_kdc.cc:121] Time spent starting KDC: real 2.069s user 0.001s sys 0.017s
16:19:51.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
16:19:51.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
16:19:51.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
16:19:51.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
16:19:51.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
16:19:51.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
16:19:51.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
16:19:51.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
16:19:51.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
16:19:51.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
16:19:51.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/krb5kdc/test-user.keytab.
16:19:51.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/krb5kdc/test-user.keytab.
16:19:51.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 23 16:19:51 dist-test-slave-fqp8 krb5kdc[22580](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790180391, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:19:51.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
16:19:51.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-23T16:19:51Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
16:19:51.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-23T16:19:51Z Disabled control of system clock
16:19:51.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.22.10.62@KRBTEST.COM; defaulting to no policy
16:19:51.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:19:51.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.22.10.62@KRBTEST.COM" created.
16:19:51.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:19:51.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.22.10.62 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/krb5kdc/kudu.keytab.
16:19:51.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.22.10.62 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/krb5kdc/kudu.keytab.
16:19:51.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.22.10.62@KRBTEST.COM; defaulting to no policy
16:19:51.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:19:51.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.22.10.62@KRBTEST.COM" created.
16:19:51.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:19:51.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.22.10.62 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/krb5kdc/kudu.keytab.
16:19:51.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.22.10.62 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/krb5kdc/kudu.keytab.
16:19:51.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:51.658716 22568 external_mini_cluster.cc:1372] Running /tmp/dist-test-tasko3a_Hu/build/tsan/bin/kudu
16:19:51.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasko3a_Hu/build/tsan/bin/kudu
16:19:51.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/master-0/wal
16:19:51.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/master-0/data
16:19:51.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:19:51.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:19:51.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:19:51.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:19:51.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:19:51.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:19:51.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/master-0/logs
16:19:51.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/master-0/data/info.pb
16:19:51.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:19:51.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:19:51.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:19:51.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:19:51.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:19:51.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:19:51.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:19:51.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:19:51.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
16:19:51.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:19:51.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:19:51.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:19:51.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.10.62:37357
16:19:51.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.10.62
16:19:51.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:19:51.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/krb5kdc/kudu.keytab
16:19:51.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.22.10.62
16:19:51.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:19:51.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
16:19:51.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
16:19:51.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
16:19:51.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.10.20:38887
16:19:51.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:19:51.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:19:51.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:19:51.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:19:51.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.10.62:37357,127.22.10.61:38773,127.22.10.60:39017
16:19:51.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:19:51.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41371/jwks.json
16:19:51.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
16:19:51.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
16:19:51.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:51.977646 22596 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
16:19:51.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:51.978380 22596 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:19:51.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:51.979107 22596 flags.cc:432] Enabled unsafe flag: --never_fsync=true
16:19:52.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:52.012745 22596 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
16:19:52.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:52.013039 22596 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
16:19:52.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:52.013437 22596 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
16:19:52.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:52.013778 22596 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
16:19:52.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:52.014166 22596 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
16:19:52.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:52.014492 22596 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
16:19:52.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:52.014735 22596 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:41371/jwks.json
16:19:52.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:52.055639 22596 master_runner.cc:387] Master server non-default flags:
16:19:52.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/master-0/logs
16:19:52.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:19:52.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:19:52.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:19:52.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.10.20:38887
16:19:52.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:19:52.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:19:52.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/master-0/data
16:19:52.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/master-0/wal
16:19:52.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
16:19:52.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:19:52.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.10.62:37357,127.22.10.61:38773,127.22.10.60:39017
16:19:52.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/krb5kdc/kudu.keytab
16:19:52.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.22.10.62
16:19:52.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:19:52.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:19:52.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:19:52.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.10.62:37357
16:19:52.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:19:52.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:19:52.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:19:52.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41371/jwks.json
16:19:52.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:19:52.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
16:19:52.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
16:19:52.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:19:52.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:19:52.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/master-0/data/info.pb
16:19:52.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.10.62
16:19:52.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:19:52.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:19:52.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:19:52.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:19:52.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:19:52.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:19:52.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:19:52.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
16:19:52.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
16:19:52.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision faa920b8364ca43fd9d419253dab85c46533dc98
16:19:52.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:19:52.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 23 Sep 2026 15:47:26 UTC on e7f111948823
16:19:52.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 13962
16:19:52.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:19:52.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:52.057271 22596 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:19:52.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:52.058496 22596 file_cache.cc:504] Constructed file cache file cache with capacity 419430
16:19:52.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:52.068395 22603 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:19:53.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:52.071183 22604 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:19:53.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:53.368489 22605 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
16:19:53.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:53.370599 22596 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
16:19:53.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:53.370998 22606 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:19:53.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:53.373515 22596 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:19:53.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:53.376994 22596 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:19:53.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:53.378543 22596 hybrid_clock.cc:648] HybridClock initialized: now 1790180393378483 us; error 47 us; skew 500 ppm
16:19:53.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:53.379655 22596 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:19:53.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 23 16:19:53 dist-test-slave-fqp8 krb5kdc[22580](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790180393, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.10.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:19:53.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:53.386410 22596 init.cc:377] Logged in from keytab as kudu/127.22.10.62@KRBTEST.COM (short username kudu)
16:19:53.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:53.392289 22596 webserver.cc:533] Webserver started at http://127.22.10.62:33841/ using document root <none> and password file <none>
16:19:53.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:53.393128 22596 fs_manager.cc:362] Metadata directory not provided
16:19:53.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:53.393306 22596 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:19:53.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:53.393697 22596 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
16:19:53.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:53.397764 22596 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/master-0/data/instance:
16:19:53.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "083466e5de4a4ff682941e6b4624d3a0"
16:19:53.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-23 16:19:53 on dist-test-slave-fqp8"
16:19:53.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:53.399133 22596 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/master-0/wal/instance:
16:19:53.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "083466e5de4a4ff682941e6b4624d3a0"
16:19:53.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-23 16:19:53 on dist-test-slave-fqp8"
16:19:53.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:53.406909 22596 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.007s sys 0.001s
16:19:53.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:53.412986 22614 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:19:53.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:53.414168 22596 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.001s
16:19:53.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:53.414445 22596 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/master-0/data,/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/master-0/wal
16:19:53.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "083466e5de4a4ff682941e6b4624d3a0"
16:19:53.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-23 16:19:53 on dist-test-slave-fqp8"
16:19:53.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:53.414731 22596 fs_report.cc:389] FS layout report
16:19:53.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:19:53.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/master-0/wal
16:19:53.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/master-0/wal
16:19:53.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/master-0/data/data
16:19:53.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:19:53.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:19:53.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:19:53.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:19:53.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:19:53.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:19:53.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:19:53.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:19:53.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:19:53.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:19:53.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:19:53.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:53.488626 22596 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
16:19:53.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:53.498062 22596 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:19:53.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:53.498587 22596 kserver.cc:163] Server-wide thread pool size limit: 3276
16:19:53.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:53.602135 22596 rpc_server.cc:307] RPC server started. Bound to: 127.22.10.62:37357
16:19:53.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:53.602274 22666 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.22.10.62:37357 every 8 connection(s)
16:19:53.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:53.605396 22596 server_base.cc:1193] Dumped server information to /tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/master-0/data/info.pb
16:19:53.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:53.609495 22568 external_mini_cluster.cc:1434] Started /tmp/dist-test-tasko3a_Hu/build/tsan/bin/kudu as pid 22596
16:19:53.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:53.610110 22568 external_mini_cluster.cc:1448] Reading /tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/master-0/wal/instance
16:19:53.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:53.613930 22667 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
16:19:53.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:53.627785 22667 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.10.62" port: 37357 } has no permanent_uuid. Determining permanent_uuid...
16:19:53.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.22.10.61@KRBTEST.COM; defaulting to no policy
16:19:53.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:19:53.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.22.10.61@KRBTEST.COM" created.
16:19:53.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 23 16:19:53 dist-test-slave-fqp8 krb5kdc[22580](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790180393, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.10.62@KRBTEST.COM for kudu/127.22.10.62@KRBTEST.COM
16:19:53.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:19:53.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.22.10.61 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/krb5kdc/kudu.keytab.
16:19:53.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.22.10.61 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/krb5kdc/kudu.keytab.
16:19:53.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:53.669842 22667 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.10.61" port: 38773 } has no permanent_uuid. Determining permanent_uuid...
16:19:53.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:53.672371 22616 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.22.10.61:38773: connect: Connection refused (error 111)
16:19:53.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:53.676291 22667 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.22.10.61:38773: Network error: Client connection negotiation failed: client connection to 127.22.10.61:38773: connect: Connection refused (error 111)
16:19:53.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.22.10.61@KRBTEST.COM; defaulting to no policy
16:19:53.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:19:53.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.22.10.61@KRBTEST.COM" created.
16:19:53.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:19:53.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.22.10.61 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/krb5kdc/kudu.keytab.
16:19:53.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.22.10.61 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/krb5kdc/kudu.keytab.
16:19:53.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:53.706445 22568 external_mini_cluster.cc:1372] Running /tmp/dist-test-tasko3a_Hu/build/tsan/bin/kudu
16:19:53.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasko3a_Hu/build/tsan/bin/kudu
16:19:53.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/master-1/wal
16:19:53.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/master-1/data
16:19:53.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:19:53.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:19:53.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:19:53.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:19:53.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:19:53.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:19:53.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/master-1/logs
16:19:53.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/master-1/data/info.pb
16:19:53.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:19:53.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:19:53.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:19:53.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:19:53.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:19:53.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:19:53.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:19:53.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:19:53.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
16:19:53.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:19:53.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:19:53.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:19:53.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.10.61:38773
16:19:53.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.10.61
16:19:53.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:19:53.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/krb5kdc/kudu.keytab
16:19:53.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.22.10.61
16:19:53.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:19:53.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
16:19:53.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
16:19:53.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
16:19:53.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.10.20:38887
16:19:53.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:19:53.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:19:53.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:19:53.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:19:53.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.10.62:37357,127.22.10.61:38773,127.22.10.60:39017
16:19:53.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:19:53.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41371/jwks.json
16:19:53.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
16:19:53.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
16:19:53.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:53.725874 22667 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.10.61" port: 38773 } attempt: 1
16:19:53.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:53.738422 22667 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.22.10.61:38773: Network error: Client connection negotiation failed: client connection to 127.22.10.61:38773: connect: Connection refused (error 111)
16:19:53.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:53.806869 22667 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.10.61" port: 38773 } attempt: 2
16:19:53.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:53.811945 22667 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.22.10.61:38773: Network error: Client connection negotiation failed: client connection to 127.22.10.61:38773: connect: Connection refused (error 111)
16:19:53.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:53.903383 22667 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.10.61" port: 38773 } attempt: 3
16:19:53.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:53.908161 22667 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.22.10.61:38773: Network error: Client connection negotiation failed: client connection to 127.22.10.61:38773: connect: Connection refused (error 111)
16:19:54.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:54.051656 22667 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.10.61" port: 38773 } attempt: 4
16:19:54.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:54.056339 22667 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.22.10.61:38773: Network error: Client connection negotiation failed: client connection to 127.22.10.61:38773: connect: Connection refused (error 111)
16:19:54.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:54.084071 22677 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
16:19:54.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:54.084604 22677 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:19:54.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:54.085165 22677 flags.cc:432] Enabled unsafe flag: --never_fsync=true
16:19:54.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:54.122587 22677 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
16:19:54.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:54.122812 22677 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
16:19:54.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:54.123085 22677 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
16:19:54.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:54.123286 22677 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
16:19:54.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:54.123472 22677 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
16:19:54.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:54.123675 22677 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
16:19:54.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:54.123822 22677 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:41371/jwks.json
16:19:54.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:54.162820 22677 master_runner.cc:387] Master server non-default flags:
16:19:54.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/master-1/logs
16:19:54.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:19:54.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:19:54.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:19:54.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.10.20:38887
16:19:54.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:19:54.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:19:54.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/master-1/data
16:19:54.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/master-1/wal
16:19:54.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
16:19:54.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:19:54.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.10.62:37357,127.22.10.61:38773,127.22.10.60:39017
16:19:54.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/krb5kdc/kudu.keytab
16:19:54.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.22.10.61
16:19:54.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:19:54.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:19:54.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:19:54.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.10.61:38773
16:19:54.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:19:54.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:19:54.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:19:54.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41371/jwks.json
16:19:54.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:19:54.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
16:19:54.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
16:19:54.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:19:54.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:19:54.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/master-1/data/info.pb
16:19:54.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.10.61
16:19:54.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:19:54.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:19:54.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:19:54.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:19:54.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:19:54.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:19:54.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:19:54.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
16:19:54.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
16:19:54.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision faa920b8364ca43fd9d419253dab85c46533dc98
16:19:54.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:19:54.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 23 Sep 2026 15:47:26 UTC on e7f111948823
16:19:54.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 13962
16:19:54.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:19:54.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:54.164034 22677 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:19:54.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:54.165079 22677 file_cache.cc:504] Constructed file cache file cache with capacity 419430
16:19:54.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:54.178377 22683 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:19:54.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:54.355810 22667 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.10.61" port: 38773 } attempt: 5
16:19:54.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:54.365592 22667 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.22.10.61:38773: Network error: Client connection negotiation failed: client connection to 127.22.10.61:38773: connect: Connection refused (error 111)
16:19:54.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:54.913293 22667 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.10.61" port: 38773 } attempt: 6
16:19:54.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:54.947866 22667 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.22.10.61:38773: Network error: Client connection negotiation failed: client connection to 127.22.10.61:38773: connect: Connection refused (error 111)
16:19:55.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:55.582716 22682 debug-util.cc:398] Leaking SignalData structure 0x7b08000372a0 after lost signal to thread 22677
16:19:55.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:54.179934 22684 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:19:55.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:55.701503 22685 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1521 milliseconds
16:19:55.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:55.698985 22677 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.519s user 0.476s sys 0.954s
16:19:55.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:55.703230 22686 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:19:55.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:55.702562 22677 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.523s user 0.477s sys 0.955s
16:19:55.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:55.703752 22677 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
16:19:55.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:55.715186 22677 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:19:55.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:55.724404 22677 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:19:55.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:55.726075 22677 hybrid_clock.cc:648] HybridClock initialized: now 1790180395725996 us; error 76 us; skew 500 ppm
16:19:55.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:55.727309 22677 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:19:55.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 23 16:19:55 dist-test-slave-fqp8 krb5kdc[22580](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790180395, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.10.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:19:55.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:55.735523 22677 init.cc:377] Logged in from keytab as kudu/127.22.10.61@KRBTEST.COM (short username kudu)
16:19:55.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:55.741156 22677 webserver.cc:533] Webserver started at http://127.22.10.61:39977/ using document root <none> and password file <none>
16:19:55.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:55.742350 22677 fs_manager.cc:362] Metadata directory not provided
16:19:55.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:55.742604 22677 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:19:55.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:55.743132 22677 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
16:19:55.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:55.749217 22677 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/master-1/data/instance:
16:19:55.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9a7438b488774217a1a7eb7c632b81f8"
16:19:55.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-23 16:19:55 on dist-test-slave-fqp8"
16:19:55.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:55.750679 22677 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/master-1/wal/instance:
16:19:55.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9a7438b488774217a1a7eb7c632b81f8"
16:19:55.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-23 16:19:55 on dist-test-slave-fqp8"
16:19:55.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:55.759203 22677 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.002s sys 0.006s
16:19:55.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:55.764583 22703 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:19:55.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:55.765700 22677 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.002s sys 0.001s
16:19:55.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:55.766057 22677 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/master-1/data,/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/master-1/wal
16:19:55.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9a7438b488774217a1a7eb7c632b81f8"
16:19:55.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-23 16:19:55 on dist-test-slave-fqp8"
16:19:55.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:55.766350 22677 fs_report.cc:389] FS layout report
16:19:55.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:19:55.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/master-1/wal
16:19:55.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/master-1/wal
16:19:55.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/master-1/data/data
16:19:55.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:19:55.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:19:55.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:19:55.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:19:55.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:19:55.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:19:55.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:19:55.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:19:55.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:19:55.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:19:55.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:19:55.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:55.835062 22677 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
16:19:55.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:55.840451 22677 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:19:55.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:55.840866 22677 kserver.cc:163] Server-wide thread pool size limit: 3276
16:19:55.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:55.932303 22677 rpc_server.cc:307] RPC server started. Bound to: 127.22.10.61:38773
16:19:55.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:55.932474 22754 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.22.10.61:38773 every 8 connection(s)
16:19:55.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:55.935315 22677 server_base.cc:1193] Dumped server information to /tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/master-1/data/info.pb
16:19:55.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:55.941398 22755 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
16:19:55.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:55.944171 22568 external_mini_cluster.cc:1434] Started /tmp/dist-test-tasko3a_Hu/build/tsan/bin/kudu as pid 22677
16:19:55.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:55.944679 22568 external_mini_cluster.cc:1448] Reading /tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/master-1/wal/instance
16:19:55.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:55.958065 22755 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.10.62" port: 37357 } has no permanent_uuid. Determining permanent_uuid...
16:19:55.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.22.10.60@KRBTEST.COM; defaulting to no policy
16:19:55.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:19:55.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.22.10.60@KRBTEST.COM" created.
16:19:55.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 23 16:19:55 dist-test-slave-fqp8 krb5kdc[22580](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790180395, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.10.61@KRBTEST.COM for kudu/127.22.10.62@KRBTEST.COM
16:19:55.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:55.997994 22755 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.10.61" port: 38773 } has no permanent_uuid. Determining permanent_uuid...
16:19:56.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:19:56.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.22.10.60 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/krb5kdc/kudu.keytab.
16:19:56.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.22.10.60 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/krb5kdc/kudu.keytab.
16:19:56.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:56.008483 22667 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.10.61" port: 38773 } attempt: 7
16:19:56.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.22.10.60@KRBTEST.COM; defaulting to no policy
16:19:56.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 23 16:19:56 dist-test-slave-fqp8 krb5kdc[22580](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790180395, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.10.61@KRBTEST.COM for kudu/127.22.10.61@KRBTEST.COM
16:19:56.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:19:56.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.22.10.60@KRBTEST.COM" created.
16:19:56.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 23 16:19:56 dist-test-slave-fqp8 krb5kdc[22580](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790180393, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.10.62@KRBTEST.COM for kudu/127.22.10.61@KRBTEST.COM
16:19:56.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:56.037041 22755 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.10.60" port: 39017 } has no permanent_uuid. Determining permanent_uuid...
16:19:56.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:56.040337 22705 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.22.10.60:39017: connect: Connection refused (error 111)
16:19:56.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:56.048122 22755 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.22.10.60:39017: Network error: Client connection negotiation failed: client connection to 127.22.10.60:39017: connect: Connection refused (error 111)
16:19:56.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:56.048237 22667 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.10.60" port: 39017 } has no permanent_uuid. Determining permanent_uuid...
16:19:56.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:19:56.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.22.10.60 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/krb5kdc/kudu.keytab.
16:19:56.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.22.10.60 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/krb5kdc/kudu.keytab.
16:19:56.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:56.053629 22667 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.22.10.60:39017: Network error: Client connection negotiation failed: client connection to 127.22.10.60:39017: connect: Connection refused (error 111)
16:19:56.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:56.055480 22568 external_mini_cluster.cc:1372] Running /tmp/dist-test-tasko3a_Hu/build/tsan/bin/kudu
16:19:56.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasko3a_Hu/build/tsan/bin/kudu
16:19:56.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/master-2/wal
16:19:56.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/master-2/data
16:19:56.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:19:56.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:19:56.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:19:56.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:19:56.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:19:56.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:19:56.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/master-2/logs
16:19:56.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/master-2/data/info.pb
16:19:56.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:19:56.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:19:56.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:19:56.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:19:56.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:19:56.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:19:56.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:19:56.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:19:56.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
16:19:56.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:19:56.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:19:56.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:19:56.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.10.60:39017
16:19:56.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.10.60
16:19:56.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:19:56.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/krb5kdc/kudu.keytab
16:19:56.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.22.10.60
16:19:56.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:19:56.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
16:19:56.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
16:19:56.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
16:19:56.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.10.20:38887
16:19:56.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:19:56.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:19:56.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:19:56.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:19:56.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.10.62:37357,127.22.10.61:38773,127.22.10.60:39017
16:19:56.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:19:56.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41371/jwks.json
16:19:56.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
16:19:56.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
16:19:56.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:56.097669 22755 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.10.60" port: 39017 } attempt: 1
16:19:56.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:56.103083 22755 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.22.10.60:39017: Network error: Client connection negotiation failed: client connection to 127.22.10.60:39017: connect: Connection refused (error 111)
16:19:56.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:56.112113 22667 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.10.60" port: 39017 } attempt: 1
16:19:56.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:56.116909 22667 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.22.10.60:39017: Network error: Client connection negotiation failed: client connection to 127.22.10.60:39017: connect: Connection refused (error 111)
16:19:56.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:56.171561 22755 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.10.60" port: 39017 } attempt: 2
16:19:56.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:56.176138 22755 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.22.10.60:39017: Network error: Client connection negotiation failed: client connection to 127.22.10.60:39017: connect: Connection refused (error 111)
16:19:56.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:56.198259 22667 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.10.60" port: 39017 } attempt: 2
16:19:56.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:56.202977 22667 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.22.10.60:39017: Network error: Client connection negotiation failed: client connection to 127.22.10.60:39017: connect: Connection refused (error 111)
16:19:56.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:56.267606 22755 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.10.60" port: 39017 } attempt: 3
16:19:56.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:56.272274 22755 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.22.10.60:39017: Network error: Client connection negotiation failed: client connection to 127.22.10.60:39017: connect: Connection refused (error 111)
16:19:56.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:56.288424 22667 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.10.60" port: 39017 } attempt: 3
16:19:56.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:56.293833 22667 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.22.10.60:39017: Network error: Client connection negotiation failed: client connection to 127.22.10.60:39017: connect: Connection refused (error 111)
16:19:56.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:56.415648 22755 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.10.60" port: 39017 } attempt: 4
16:19:56.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:56.418587 22768 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
16:19:56.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:56.419387 22768 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:19:56.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:56.420356 22768 flags.cc:432] Enabled unsafe flag: --never_fsync=true
16:19:56.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:56.420341 22755 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.22.10.60:39017: Network error: Client connection negotiation failed: client connection to 127.22.10.60:39017: connect: Connection refused (error 111)
16:19:56.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:56.434233 22667 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.10.60" port: 39017 } attempt: 4
16:19:56.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:56.438700 22667 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.22.10.60:39017: Network error: Client connection negotiation failed: client connection to 127.22.10.60:39017: connect: Connection refused (error 111)
16:19:56.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:56.470099 22768 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
16:19:56.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:56.470328 22768 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
16:19:56.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:56.470597 22768 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
16:19:56.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:56.470794 22768 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
16:19:56.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:56.470993 22768 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
16:19:56.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:56.471184 22768 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
16:19:56.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:56.471330 22768 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:41371/jwks.json
16:19:56.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:56.517642 22768 master_runner.cc:387] Master server non-default flags:
16:19:56.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/master-2/logs
16:19:56.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:19:56.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:19:56.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:19:56.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.10.20:38887
16:19:56.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:19:56.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:19:56.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/master-2/data
16:19:56.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/master-2/wal
16:19:56.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
16:19:56.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:19:56.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.10.62:37357,127.22.10.61:38773,127.22.10.60:39017
16:19:56.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/krb5kdc/kudu.keytab
16:19:56.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.22.10.60
16:19:56.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:19:56.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:19:56.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:19:56.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.10.60:39017
16:19:56.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:19:56.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:19:56.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:19:56.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41371/jwks.json
16:19:56.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:19:56.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
16:19:56.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
16:19:56.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:19:56.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:19:56.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/master-2/data/info.pb
16:19:56.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.10.60
16:19:56.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:19:56.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:19:56.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:19:56.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:19:56.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:19:56.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:19:56.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:19:56.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
16:19:56.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
16:19:56.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision faa920b8364ca43fd9d419253dab85c46533dc98
16:19:56.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:19:56.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 23 Sep 2026 15:47:26 UTC on e7f111948823
16:19:56.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 13962
16:19:56.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:19:56.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:56.519284 22768 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:19:56.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:56.520424 22768 file_cache.cc:504] Constructed file cache file cache with capacity 419430
16:19:56.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:56.533054 22774 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:19:56.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:56.719710 22755 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.10.60" port: 39017 } attempt: 5
16:19:56.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:56.722153 22667 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.10.60" port: 39017 } attempt: 5
16:19:56.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:56.740235 22667 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.22.10.60:39017: Network error: Client connection negotiation failed: client connection to 127.22.10.60:39017: connect: Connection refused (error 111)
16:19:56.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:56.750082 22755 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.22.10.60:39017: Network error: Client connection negotiation failed: client connection to 127.22.10.60:39017: connect: Connection refused (error 111)
16:19:57.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:57.292824 22667 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.10.60" port: 39017 } attempt: 6
16:19:57.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:57.297683 22755 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.10.60" port: 39017 } attempt: 6
16:19:57.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:57.321286 22667 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.22.10.60:39017: Network error: Client connection negotiation failed: client connection to 127.22.10.60:39017: connect: Connection refused (error 111)
16:19:57.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:57.324174 22755 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.22.10.60:39017: Network error: Client connection negotiation failed: client connection to 127.22.10.60:39017: connect: Connection refused (error 111)
16:19:57.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:56.534600 22775 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:19:57.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:57.880731 22777 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:19:57.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:57.885598 22776 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1345 milliseconds
16:19:57.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:57.885715 22768 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
16:19:57.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:57.887091 22768 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:19:57.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:57.890622 22768 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:19:57.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:57.892069 22768 hybrid_clock.cc:648] HybridClock initialized: now 1790180397892023 us; error 56 us; skew 500 ppm
16:19:57.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:57.892834 22768 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:19:57.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 23 16:19:57 dist-test-slave-fqp8 krb5kdc[22580](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790180397, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.10.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:19:57.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:57.899982 22768 init.cc:377] Logged in from keytab as kudu/127.22.10.60@KRBTEST.COM (short username kudu)
16:19:57.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:57.907325 22768 webserver.cc:533] Webserver started at http://127.22.10.60:35981/ using document root <none> and password file <none>
16:19:57.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:57.908236 22768 fs_manager.cc:362] Metadata directory not provided
16:19:57.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:57.908465 22768 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:19:57.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:57.909047 22768 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
16:19:57.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:57.914049 22768 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/master-2/data/instance:
16:19:57.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fc4bbd8cf09e438895593b7651c63f06"
16:19:57.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-23 16:19:57 on dist-test-slave-fqp8"
16:19:57.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:57.915089 22768 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/master-2/wal/instance:
16:19:57.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fc4bbd8cf09e438895593b7651c63f06"
16:19:57.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-23 16:19:57 on dist-test-slave-fqp8"
16:19:57.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:57.923429 22768 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.000s sys 0.008s
16:19:57.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:57.930227 22791 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:19:57.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:57.931945 22768 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.003s sys 0.001s
16:19:57.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:57.932632 22768 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/master-2/data,/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/master-2/wal
16:19:57.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fc4bbd8cf09e438895593b7651c63f06"
16:19:57.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-23 16:19:57 on dist-test-slave-fqp8"
16:19:57.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:57.933030 22768 fs_report.cc:389] FS layout report
16:19:57.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:19:57.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/master-2/wal
16:19:57.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/master-2/wal
16:19:57.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/master-2/data/data
16:19:57.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:19:57.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:19:57.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:19:57.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:19:57.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:19:57.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:19:57.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:19:57.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:19:57.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:19:57.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:19:57.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:19:57.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:57.993260 22768 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
16:19:58.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:57.999709 22768 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:19:58.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:58.000272 22768 kserver.cc:163] Server-wide thread pool size limit: 3276
16:19:58.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:58.095960 22768 rpc_server.cc:307] RPC server started. Bound to: 127.22.10.60:39017
16:19:58.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:58.096168 22842 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.22.10.60:39017 every 8 connection(s)
16:19:58.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:58.098955 22768 server_base.cc:1193] Dumped server information to /tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/master-2/data/info.pb
16:19:58.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:58.104064 22568 external_mini_cluster.cc:1434] Started /tmp/dist-test-tasko3a_Hu/build/tsan/bin/kudu as pid 22768
16:19:58.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:58.104620 22568 external_mini_cluster.cc:1448] Reading /tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/master-2/wal/instance
16:19:58.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:58.106760 22843 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
16:19:58.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:58.123755 22843 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.10.62" port: 37357 } has no permanent_uuid. Determining permanent_uuid...
16:19:58.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 23 16:19:58 dist-test-slave-fqp8 krb5kdc[22580](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790180391, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.22.10.62@KRBTEST.COM
16:19:58.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 23 16:19:58 dist-test-slave-fqp8 krb5kdc[22580](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790180397, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.10.60@KRBTEST.COM for kudu/127.22.10.62@KRBTEST.COM
16:19:58.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 23 16:19:58 dist-test-slave-fqp8 krb5kdc[22580](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790180391, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.22.10.61@KRBTEST.COM
16:19:58.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:58.162674 22843 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.10.61" port: 38773 } has no permanent_uuid. Determining permanent_uuid...
16:19:58.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 23 16:19:58 dist-test-slave-fqp8 krb5kdc[22580](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790180397, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.10.60@KRBTEST.COM for kudu/127.22.10.61@KRBTEST.COM
16:19:58.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 23 16:19:58 dist-test-slave-fqp8 krb5kdc[22580](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790180391, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.22.10.60@KRBTEST.COM
16:19:58.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:58.185639 22843 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.10.60" port: 39017 } has no permanent_uuid. Determining permanent_uuid...
16:19:58.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 23 16:19:58 dist-test-slave-fqp8 krb5kdc[22580](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790180397, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.10.60@KRBTEST.COM for kudu/127.22.10.60@KRBTEST.COM
16:19:58.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:58.222308 22843 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P fc4bbd8cf09e438895593b7651c63f06: Bootstrap starting.
16:19:58.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:58.229108 22843 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P fc4bbd8cf09e438895593b7651c63f06: Neither blocks nor log segments found. Creating new log.
16:19:58.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:58.231369 22843 log.cc:826] T 00000000000000000000000000000000 P fc4bbd8cf09e438895593b7651c63f06: Log is configured to *not* fsync() on all Append() calls
16:19:58.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:58.236754 22843 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P fc4bbd8cf09e438895593b7651c63f06: No bootstrap required, opened a new log
16:19:58.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:58.257599 22843 raft_consensus.cc:359] T 00000000000000000000000000000000 P fc4bbd8cf09e438895593b7651c63f06 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "083466e5de4a4ff682941e6b4624d3a0" member_type: VOTER last_known_addr { host: "127.22.10.62" port: 37357 } } peers { permanent_uuid: "9a7438b488774217a1a7eb7c632b81f8" member_type: VOTER last_known_addr { host: "127.22.10.61" port: 38773 } } peers { permanent_uuid: "fc4bbd8cf09e438895593b7651c63f06" member_type: VOTER last_known_addr { host: "127.22.10.60" port: 39017 } }
16:19:58.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:58.258376 22843 raft_consensus.cc:385] T 00000000000000000000000000000000 P fc4bbd8cf09e438895593b7651c63f06 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
16:19:58.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:58.258674 22843 raft_consensus.cc:740] T 00000000000000000000000000000000 P fc4bbd8cf09e438895593b7651c63f06 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: fc4bbd8cf09e438895593b7651c63f06, State: Initialized, Role: FOLLOWER
16:19:58.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:58.259627 22843 consensus_queue.cc:260] T 00000000000000000000000000000000 P fc4bbd8cf09e438895593b7651c63f06 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "083466e5de4a4ff682941e6b4624d3a0" member_type: VOTER last_known_addr { host: "127.22.10.62" port: 37357 } } peers { permanent_uuid: "9a7438b488774217a1a7eb7c632b81f8" member_type: VOTER last_known_addr { host: "127.22.10.61" port: 38773 } } peers { permanent_uuid: "fc4bbd8cf09e438895593b7651c63f06" member_type: VOTER last_known_addr { host: "127.22.10.60" port: 39017 } }
16:19:58.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:58.262334 22855 sys_catalog.cc:455] T 00000000000000000000000000000000 P fc4bbd8cf09e438895593b7651c63f06 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "083466e5de4a4ff682941e6b4624d3a0" member_type: VOTER last_known_addr { host: "127.22.10.62" port: 37357 } } peers { permanent_uuid: "9a7438b488774217a1a7eb7c632b81f8" member_type: VOTER last_known_addr { host: "127.22.10.61" port: 38773 } } peers { permanent_uuid: "fc4bbd8cf09e438895593b7651c63f06" member_type: VOTER last_known_addr { host: "127.22.10.60" port: 39017 } } }
16:19:58.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:58.262899 22855 sys_catalog.cc:458] T 00000000000000000000000000000000 P fc4bbd8cf09e438895593b7651c63f06 [sys.catalog]: This master's current role is: FOLLOWER
16:19:58.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:58.263747 22843 sys_catalog.cc:565] T 00000000000000000000000000000000 P fc4bbd8cf09e438895593b7651c63f06 [sys.catalog]: configured and running, proceeding with master startup.
16:19:58.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:58.294297 22866 catalog_manager.cc:1607] T 00000000000000000000000000000000 P fc4bbd8cf09e438895593b7651c63f06: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:19:58.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:58.294660 22866 catalog_manager.cc:896] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:19:58.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:58.289251 22768 master_runner.cc:187] Error getting master registration for 127.22.10.62:37357: OK, instance_id { permanent_uuid: "083466e5de4a4ff682941e6b4624d3a0" instance_seqno: 1790180393573933 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
16:19:58.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:58.311993 22768 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
16:19:58.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:58.354871 22667 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.10.60" port: 39017 } attempt: 7
16:19:58.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 23 16:19:58 dist-test-slave-fqp8 krb5kdc[22580](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790180393, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.10.62@KRBTEST.COM for kudu/127.22.10.60@KRBTEST.COM
16:19:58.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:58.384773 22755 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.10.60" port: 39017 } attempt: 7
16:19:58.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:58.387359 22667 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 083466e5de4a4ff682941e6b4624d3a0: Bootstrap starting.
16:19:58.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:58.396659 22667 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 083466e5de4a4ff682941e6b4624d3a0: Neither blocks nor log segments found. Creating new log.
16:19:58.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:58.399462 22667 log.cc:826] T 00000000000000000000000000000000 P 083466e5de4a4ff682941e6b4624d3a0: Log is configured to *not* fsync() on all Append() calls
16:19:58.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 23 16:19:58 dist-test-slave-fqp8 krb5kdc[22580](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790180395, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.10.61@KRBTEST.COM for kudu/127.22.10.60@KRBTEST.COM
16:19:58.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:58.409772 22667 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 083466e5de4a4ff682941e6b4624d3a0: No bootstrap required, opened a new log
16:19:58.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:58.427630 22755 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9a7438b488774217a1a7eb7c632b81f8: Bootstrap starting.
16:19:58.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:58.437749 22755 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 9a7438b488774217a1a7eb7c632b81f8: Neither blocks nor log segments found. Creating new log.
16:19:58.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:58.438761 22667 raft_consensus.cc:359] T 00000000000000000000000000000000 P 083466e5de4a4ff682941e6b4624d3a0 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "083466e5de4a4ff682941e6b4624d3a0" member_type: VOTER last_known_addr { host: "127.22.10.62" port: 37357 } } peers { permanent_uuid: "9a7438b488774217a1a7eb7c632b81f8" member_type: VOTER last_known_addr { host: "127.22.10.61" port: 38773 } } peers { permanent_uuid: "fc4bbd8cf09e438895593b7651c63f06" member_type: VOTER last_known_addr { host: "127.22.10.60" port: 39017 } }
16:19:58.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:58.439728 22667 raft_consensus.cc:385] T 00000000000000000000000000000000 P 083466e5de4a4ff682941e6b4624d3a0 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
16:19:58.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:58.440157 22667 raft_consensus.cc:740] T 00000000000000000000000000000000 P 083466e5de4a4ff682941e6b4624d3a0 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 083466e5de4a4ff682941e6b4624d3a0, State: Initialized, Role: FOLLOWER
16:19:58.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:58.440400 22755 log.cc:826] T 00000000000000000000000000000000 P 9a7438b488774217a1a7eb7c632b81f8: Log is configured to *not* fsync() on all Append() calls
16:19:58.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:58.441412 22667 consensus_queue.cc:260] T 00000000000000000000000000000000 P 083466e5de4a4ff682941e6b4624d3a0 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "083466e5de4a4ff682941e6b4624d3a0" member_type: VOTER last_known_addr { host: "127.22.10.62" port: 37357 } } peers { permanent_uuid: "9a7438b488774217a1a7eb7c632b81f8" member_type: VOTER last_known_addr { host: "127.22.10.61" port: 38773 } } peers { permanent_uuid: "fc4bbd8cf09e438895593b7651c63f06" member_type: VOTER last_known_addr { host: "127.22.10.60" port: 39017 } }
16:19:58.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:58.456632 22667 sys_catalog.cc:565] T 00000000000000000000000000000000 P 083466e5de4a4ff682941e6b4624d3a0 [sys.catalog]: configured and running, proceeding with master startup.
16:19:58.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:58.457705 22755 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9a7438b488774217a1a7eb7c632b81f8: No bootstrap required, opened a new log
16:19:58.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:58.462107 22873 sys_catalog.cc:455] T 00000000000000000000000000000000 P 083466e5de4a4ff682941e6b4624d3a0 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "083466e5de4a4ff682941e6b4624d3a0" member_type: VOTER last_known_addr { host: "127.22.10.62" port: 37357 } } peers { permanent_uuid: "9a7438b488774217a1a7eb7c632b81f8" member_type: VOTER last_known_addr { host: "127.22.10.61" port: 38773 } } peers { permanent_uuid: "fc4bbd8cf09e438895593b7651c63f06" member_type: VOTER last_known_addr { host: "127.22.10.60" port: 39017 } } }
16:19:58.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:58.463071 22873 sys_catalog.cc:458] T 00000000000000000000000000000000 P 083466e5de4a4ff682941e6b4624d3a0 [sys.catalog]: This master's current role is: FOLLOWER
16:19:58.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:58.490415 22755 raft_consensus.cc:359] T 00000000000000000000000000000000 P 9a7438b488774217a1a7eb7c632b81f8 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "083466e5de4a4ff682941e6b4624d3a0" member_type: VOTER last_known_addr { host: "127.22.10.62" port: 37357 } } peers { permanent_uuid: "9a7438b488774217a1a7eb7c632b81f8" member_type: VOTER last_known_addr { host: "127.22.10.61" port: 38773 } } peers { permanent_uuid: "fc4bbd8cf09e438895593b7651c63f06" member_type: VOTER last_known_addr { host: "127.22.10.60" port: 39017 } }
16:19:58.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:58.491606 22755 raft_consensus.cc:385] T 00000000000000000000000000000000 P 9a7438b488774217a1a7eb7c632b81f8 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
16:19:58.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:58.492089 22755 raft_consensus.cc:740] T 00000000000000000000000000000000 P 9a7438b488774217a1a7eb7c632b81f8 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9a7438b488774217a1a7eb7c632b81f8, State: Initialized, Role: FOLLOWER
16:19:58.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:58.493368 22755 consensus_queue.cc:260] T 00000000000000000000000000000000 P 9a7438b488774217a1a7eb7c632b81f8 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "083466e5de4a4ff682941e6b4624d3a0" member_type: VOTER last_known_addr { host: "127.22.10.62" port: 37357 } } peers { permanent_uuid: "9a7438b488774217a1a7eb7c632b81f8" member_type: VOTER last_known_addr { host: "127.22.10.61" port: 38773 } } peers { permanent_uuid: "fc4bbd8cf09e438895593b7651c63f06" member_type: VOTER last_known_addr { host: "127.22.10.60" port: 39017 } }
16:19:58.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:58.506762 22755 sys_catalog.cc:565] T 00000000000000000000000000000000 P 9a7438b488774217a1a7eb7c632b81f8 [sys.catalog]: configured and running, proceeding with master startup.
16:19:58.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:58.506856 22797 tablet.cc:2433] T 00000000000000000000000000000000 P fc4bbd8cf09e438895593b7651c63f06: Can't schedule compaction. Clean time has not been advanced past its initial value.
16:19:58.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:58.510156 22620 tablet.cc:2433] T 00000000000000000000000000000000 P 083466e5de4a4ff682941e6b4624d3a0: Can't schedule compaction. Clean time has not been advanced past its initial value.
16:19:58.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:58.502115 22876 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9a7438b488774217a1a7eb7c632b81f8 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "083466e5de4a4ff682941e6b4624d3a0" member_type: VOTER last_known_addr { host: "127.22.10.62" port: 37357 } } peers { permanent_uuid: "9a7438b488774217a1a7eb7c632b81f8" member_type: VOTER last_known_addr { host: "127.22.10.61" port: 38773 } } peers { permanent_uuid: "fc4bbd8cf09e438895593b7651c63f06" member_type: VOTER last_known_addr { host: "127.22.10.60" port: 39017 } } }
16:19:58.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:58.511474 22876 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9a7438b488774217a1a7eb7c632b81f8 [sys.catalog]: This master's current role is: FOLLOWER
16:19:58.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:58.587174 22896 catalog_manager.cc:1607] T 00000000000000000000000000000000 P 9a7438b488774217a1a7eb7c632b81f8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:19:58.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:58.587873 22896 catalog_manager.cc:896] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:19:58.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:58.601038 22709 tablet.cc:2433] T 00000000000000000000000000000000 P 9a7438b488774217a1a7eb7c632b81f8: Can't schedule compaction. Clean time has not been advanced past its initial value.
16:19:58.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:58.563045 22677 master_runner.cc:187] Error getting master registration for 127.22.10.62:37357: OK, instance_id { permanent_uuid: "083466e5de4a4ff682941e6b4624d3a0" instance_seqno: 1790180393573933 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
16:19:58.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:58.604406 22677 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
16:19:58.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:58.608664 22596 master_runner.cc:264] No leader master found from master 083466e5de4a4ff682941e6b4624d3a0
16:19:58.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:58.608960 22596 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
16:19:58.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:58.639956 22894 catalog_manager.cc:1607] T 00000000000000000000000000000000 P 083466e5de4a4ff682941e6b4624d3a0: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:19:58.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:58.640666 22894 catalog_manager.cc:896] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:19:58.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:58.682540 22855 raft_consensus.cc:493] T 00000000000000000000000000000000 P fc4bbd8cf09e438895593b7651c63f06 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
16:19:58.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:58.683099 22855 raft_consensus.cc:515] T 00000000000000000000000000000000 P fc4bbd8cf09e438895593b7651c63f06 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "083466e5de4a4ff682941e6b4624d3a0" member_type: VOTER last_known_addr { host: "127.22.10.62" port: 37357 } } peers { permanent_uuid: "9a7438b488774217a1a7eb7c632b81f8" member_type: VOTER last_known_addr { host: "127.22.10.61" port: 38773 } } peers { permanent_uuid: "fc4bbd8cf09e438895593b7651c63f06" member_type: VOTER last_known_addr { host: "127.22.10.60" port: 39017 } }
16:19:58.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:58.685622 22855 leader_election.cc:290] T 00000000000000000000000000000000 P fc4bbd8cf09e438895593b7651c63f06 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 083466e5de4a4ff682941e6b4624d3a0 (127.22.10.62:37357), 9a7438b488774217a1a7eb7c632b81f8 (127.22.10.61:38773)
16:19:58.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:58.687269 22730 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "fc4bbd8cf09e438895593b7651c63f06" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9a7438b488774217a1a7eb7c632b81f8" is_pre_election: true
16:19:58.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:58.687559 22642 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "fc4bbd8cf09e438895593b7651c63f06" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "083466e5de4a4ff682941e6b4624d3a0" is_pre_election: true
16:19:58.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:58.688328 22642 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 083466e5de4a4ff682941e6b4624d3a0 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate fc4bbd8cf09e438895593b7651c63f06 in term 0.
16:19:58.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:58.688253 22730 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 9a7438b488774217a1a7eb7c632b81f8 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate fc4bbd8cf09e438895593b7651c63f06 in term 0.
16:19:58.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:58.689898 22792 leader_election.cc:304] T 00000000000000000000000000000000 P fc4bbd8cf09e438895593b7651c63f06 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 083466e5de4a4ff682941e6b4624d3a0, fc4bbd8cf09e438895593b7651c63f06; no voters:
16:19:58.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:58.690881 22855 raft_consensus.cc:2804] T 00000000000000000000000000000000 P fc4bbd8cf09e438895593b7651c63f06 [term 0 FOLLOWER]: Leader pre-election won for term 1
16:19:58.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:58.691249 22855 raft_consensus.cc:493] T 00000000000000000000000000000000 P fc4bbd8cf09e438895593b7651c63f06 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
16:19:58.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:58.691570 22855 raft_consensus.cc:3060] T 00000000000000000000000000000000 P fc4bbd8cf09e438895593b7651c63f06 [term 0 FOLLOWER]: Advancing to term 1
16:19:58.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:58.707736 22855 raft_consensus.cc:515] T 00000000000000000000000000000000 P fc4bbd8cf09e438895593b7651c63f06 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "083466e5de4a4ff682941e6b4624d3a0" member_type: VOTER last_known_addr { host: "127.22.10.62" port: 37357 } } peers { permanent_uuid: "9a7438b488774217a1a7eb7c632b81f8" member_type: VOTER last_known_addr { host: "127.22.10.61" port: 38773 } } peers { permanent_uuid: "fc4bbd8cf09e438895593b7651c63f06" member_type: VOTER last_known_addr { host: "127.22.10.60" port: 39017 } }
16:19:58.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:58.710489 22642 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "fc4bbd8cf09e438895593b7651c63f06" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "083466e5de4a4ff682941e6b4624d3a0"
16:19:58.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:58.711023 22642 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 083466e5de4a4ff682941e6b4624d3a0 [term 0 FOLLOWER]: Advancing to term 1
16:19:58.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:58.718053 22642 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 083466e5de4a4ff682941e6b4624d3a0 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate fc4bbd8cf09e438895593b7651c63f06 in term 1.
16:19:58.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:58.727556 22792 leader_election.cc:304] T 00000000000000000000000000000000 P fc4bbd8cf09e438895593b7651c63f06 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 083466e5de4a4ff682941e6b4624d3a0, fc4bbd8cf09e438895593b7651c63f06; no voters:
16:19:58.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:58.750748 22730 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "fc4bbd8cf09e438895593b7651c63f06" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9a7438b488774217a1a7eb7c632b81f8"
16:19:58.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:58.751379 22730 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 9a7438b488774217a1a7eb7c632b81f8 [term 0 FOLLOWER]: Advancing to term 1
16:19:58.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:58.765316 22730 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 9a7438b488774217a1a7eb7c632b81f8 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate fc4bbd8cf09e438895593b7651c63f06 in term 1.
16:19:58.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:58.784511 22899 raft_consensus.cc:2804] T 00000000000000000000000000000000 P fc4bbd8cf09e438895593b7651c63f06 [term 1 FOLLOWER]: Leader election won for term 1
16:19:58.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:58.785331 22792 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.RequestConsensusVote -> {remote=127.22.10.62:37357, user_credentials={real_user=slave}} blocked reactor thread for 57862.4us
16:19:58.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:58.785892 22855 leader_election.cc:290] T 00000000000000000000000000000000 P fc4bbd8cf09e438895593b7651c63f06 [CANDIDATE]: Term 1 election: Requested vote from peers 083466e5de4a4ff682941e6b4624d3a0 (127.22.10.62:37357), 9a7438b488774217a1a7eb7c632b81f8 (127.22.10.61:38773)
16:19:58.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:58.788877 22899 raft_consensus.cc:697] T 00000000000000000000000000000000 P fc4bbd8cf09e438895593b7651c63f06 [term 1 LEADER]: Becoming Leader. State: Replica: fc4bbd8cf09e438895593b7651c63f06, State: Running, Role: LEADER
16:19:58.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:58.795282 22899 consensus_queue.cc:237] T 00000000000000000000000000000000 P fc4bbd8cf09e438895593b7651c63f06 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "083466e5de4a4ff682941e6b4624d3a0" member_type: VOTER last_known_addr { host: "127.22.10.62" port: 37357 } } peers { permanent_uuid: "9a7438b488774217a1a7eb7c632b81f8" member_type: VOTER last_known_addr { host: "127.22.10.61" port: 38773 } } peers { permanent_uuid: "fc4bbd8cf09e438895593b7651c63f06" member_type: VOTER last_known_addr { host: "127.22.10.60" port: 39017 } }
16:19:58.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:58.816653 22855 sys_catalog.cc:455] T 00000000000000000000000000000000 P fc4bbd8cf09e438895593b7651c63f06 [sys.catalog]: SysCatalogTable state changed. Reason: New leader fc4bbd8cf09e438895593b7651c63f06. Latest consensus state: current_term: 1 leader_uuid: "fc4bbd8cf09e438895593b7651c63f06" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "083466e5de4a4ff682941e6b4624d3a0" member_type: VOTER last_known_addr { host: "127.22.10.62" port: 37357 } } peers { permanent_uuid: "9a7438b488774217a1a7eb7c632b81f8" member_type: VOTER last_known_addr { host: "127.22.10.61" port: 38773 } } peers { permanent_uuid: "fc4bbd8cf09e438895593b7651c63f06" member_type: VOTER last_known_addr { host: "127.22.10.60" port: 39017 } } }
16:19:58.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:58.817816 22855 sys_catalog.cc:458] T 00000000000000000000000000000000 P fc4bbd8cf09e438895593b7651c63f06 [sys.catalog]: This master's current role is: LEADER
16:19:58.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:58.827754 22902 catalog_manager.cc:1524] Loading table and tablet metadata into memory...
16:19:58.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:58.845366 22902 catalog_manager.cc:1533] Initializing Kudu cluster ID...
16:19:58.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:58.872433 22730 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 9a7438b488774217a1a7eb7c632b81f8 [term 1 FOLLOWER]: Refusing update from remote peer fc4bbd8cf09e438895593b7651c63f06: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
16:19:58.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:58.875386 22855 consensus_queue.cc:1048] T 00000000000000000000000000000000 P fc4bbd8cf09e438895593b7651c63f06 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9a7438b488774217a1a7eb7c632b81f8" member_type: VOTER last_known_addr { host: "127.22.10.61" port: 38773 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
16:19:58.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:58.898001 22642 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 083466e5de4a4ff682941e6b4624d3a0 [term 1 FOLLOWER]: Refusing update from remote peer fc4bbd8cf09e438895593b7651c63f06: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
16:19:58.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:58.900241 22855 consensus_queue.cc:1048] T 00000000000000000000000000000000 P fc4bbd8cf09e438895593b7651c63f06 [LEADER]: Connected to new peer: Peer: permanent_uuid: "083466e5de4a4ff682941e6b4624d3a0" member_type: VOTER last_known_addr { host: "127.22.10.62" port: 37357 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
16:19:58.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:58.930404 22873 sys_catalog.cc:455] T 00000000000000000000000000000000 P 083466e5de4a4ff682941e6b4624d3a0 [sys.catalog]: SysCatalogTable state changed. Reason: New leader fc4bbd8cf09e438895593b7651c63f06. Latest consensus state: current_term: 1 leader_uuid: "fc4bbd8cf09e438895593b7651c63f06" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "083466e5de4a4ff682941e6b4624d3a0" member_type: VOTER last_known_addr { host: "127.22.10.62" port: 37357 } } peers { permanent_uuid: "9a7438b488774217a1a7eb7c632b81f8" member_type: VOTER last_known_addr { host: "127.22.10.61" port: 38773 } } peers { permanent_uuid: "fc4bbd8cf09e438895593b7651c63f06" member_type: VOTER last_known_addr { host: "127.22.10.60" port: 39017 } } }
16:19:58.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:58.931098 22873 sys_catalog.cc:458] T 00000000000000000000000000000000 P 083466e5de4a4ff682941e6b4624d3a0 [sys.catalog]: This master's current role is: FOLLOWER
16:19:58.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:58.939260 22903 mvcc.cc:204] Tried to move back new op lower bound from 7332578913760907264 to 7332578913481969664. Current Snapshot: MvccSnapshot[applied={T|T < 7332578913760907264}]
16:19:58.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:58.942759 22899 sys_catalog.cc:455] T 00000000000000000000000000000000 P fc4bbd8cf09e438895593b7651c63f06 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "fc4bbd8cf09e438895593b7651c63f06" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "083466e5de4a4ff682941e6b4624d3a0" member_type: VOTER last_known_addr { host: "127.22.10.62" port: 37357 } } peers { permanent_uuid: "9a7438b488774217a1a7eb7c632b81f8" member_type: VOTER last_known_addr { host: "127.22.10.61" port: 38773 } } peers { permanent_uuid: "fc4bbd8cf09e438895593b7651c63f06" member_type: VOTER last_known_addr { host: "127.22.10.60" port: 39017 } } }
16:19:58.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:58.943153 22904 mvcc.cc:204] Tried to move back new op lower bound from 7332578913760907264 to 7332578913481969664. Current Snapshot: MvccSnapshot[applied={T|T < 7332578913760907264}]
16:19:58.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:58.943463 22899 sys_catalog.cc:458] T 00000000000000000000000000000000 P fc4bbd8cf09e438895593b7651c63f06 [sys.catalog]: This master's current role is: LEADER
16:19:58.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:58.948712 22902 catalog_manager.cc:1396] Generated new cluster ID: 2b04f3c8aee34f46a1410a54279e8a40
16:19:58.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:58.949041 22902 catalog_manager.cc:1544] Initializing Kudu internal certificate authority...
16:19:58.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:58.952098 22876 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9a7438b488774217a1a7eb7c632b81f8 [sys.catalog]: SysCatalogTable state changed. Reason: New leader fc4bbd8cf09e438895593b7651c63f06. Latest consensus state: current_term: 1 leader_uuid: "fc4bbd8cf09e438895593b7651c63f06" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "083466e5de4a4ff682941e6b4624d3a0" member_type: VOTER last_known_addr { host: "127.22.10.62" port: 37357 } } peers { permanent_uuid: "9a7438b488774217a1a7eb7c632b81f8" member_type: VOTER last_known_addr { host: "127.22.10.61" port: 38773 } } peers { permanent_uuid: "fc4bbd8cf09e438895593b7651c63f06" member_type: VOTER last_known_addr { host: "127.22.10.60" port: 39017 } } }
16:19:58.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:58.952739 22876 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9a7438b488774217a1a7eb7c632b81f8 [sys.catalog]: This master's current role is: FOLLOWER
16:19:58.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:58.958489 22855 sys_catalog.cc:455] T 00000000000000000000000000000000 P fc4bbd8cf09e438895593b7651c63f06 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "fc4bbd8cf09e438895593b7651c63f06" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "083466e5de4a4ff682941e6b4624d3a0" member_type: VOTER last_known_addr { host: "127.22.10.62" port: 37357 } } peers { permanent_uuid: "9a7438b488774217a1a7eb7c632b81f8" member_type: VOTER last_known_addr { host: "127.22.10.61" port: 38773 } } peers { permanent_uuid: "fc4bbd8cf09e438895593b7651c63f06" member_type: VOTER last_known_addr { host: "127.22.10.60" port: 39017 } } }
16:19:58.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:58.959237 22855 sys_catalog.cc:458] T 00000000000000000000000000000000 P fc4bbd8cf09e438895593b7651c63f06 [sys.catalog]: This master's current role is: LEADER
16:19:58.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:58.961611 22873 sys_catalog.cc:455] T 00000000000000000000000000000000 P 083466e5de4a4ff682941e6b4624d3a0 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "fc4bbd8cf09e438895593b7651c63f06" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "083466e5de4a4ff682941e6b4624d3a0" member_type: VOTER last_known_addr { host: "127.22.10.62" port: 37357 } } peers { permanent_uuid: "9a7438b488774217a1a7eb7c632b81f8" member_type: VOTER last_known_addr { host: "127.22.10.61" port: 38773 } } peers { permanent_uuid: "fc4bbd8cf09e438895593b7651c63f06" member_type: VOTER last_known_addr { host: "127.22.10.60" port: 39017 } } }
16:19:58.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:58.962333 22873 sys_catalog.cc:458] T 00000000000000000000000000000000 P 083466e5de4a4ff682941e6b4624d3a0 [sys.catalog]: This master's current role is: FOLLOWER
16:19:58.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:58.963387 22905 mvcc.cc:204] Tried to move back new op lower bound from 7332578913760907264 to 7332578913481969664. Current Snapshot: MvccSnapshot[applied={T|T < 7332578913760907264}]
16:19:58.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:58.967100 22876 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9a7438b488774217a1a7eb7c632b81f8 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "fc4bbd8cf09e438895593b7651c63f06" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "083466e5de4a4ff682941e6b4624d3a0" member_type: VOTER last_known_addr { host: "127.22.10.62" port: 37357 } } peers { permanent_uuid: "9a7438b488774217a1a7eb7c632b81f8" member_type: VOTER last_known_addr { host: "127.22.10.61" port: 38773 } } peers { permanent_uuid: "fc4bbd8cf09e438895593b7651c63f06" member_type: VOTER last_known_addr { host: "127.22.10.60" port: 39017 } } }
16:19:58.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:58.967751 22876 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9a7438b488774217a1a7eb7c632b81f8 [sys.catalog]: This master's current role is: FOLLOWER
16:19:59.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:59.013235 22902 catalog_manager.cc:1419] Generated new certificate authority record
16:19:59.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:59.015625 22902 catalog_manager.cc:1553] Loading token signing keys...
16:19:59.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:59.090073 22902 catalog_manager.cc:6219] T 00000000000000000000000000000000 P fc4bbd8cf09e438895593b7651c63f06: Generated new TSK 0
16:19:59.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:59.091636 22902 catalog_manager.cc:1563] Initializing in-progress tserver states...
16:19:59.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.22.10.1@KRBTEST.COM; defaulting to no policy
16:19:59.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:19:59.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.22.10.1@KRBTEST.COM" created.
16:19:59.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:19:59.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.22.10.1 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/krb5kdc/kudu.keytab.
16:19:59.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.22.10.1 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/krb5kdc/kudu.keytab.
16:19:59.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:59.233198 22568 mini_kdc.cc:268] Time spent creating service keytab for kudu/127.22.10.1: real 0.102s user 0.007s sys 0.008s
16:19:59.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.22.10.1@KRBTEST.COM; defaulting to no policy
16:19:59.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:19:59.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.22.10.1@KRBTEST.COM" created.
16:19:59.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:19:59.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.22.10.1 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/krb5kdc/kudu.keytab.
16:19:59.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.22.10.1 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/krb5kdc/kudu.keytab.
16:19:59.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:59.319808 22568 external_mini_cluster.cc:1372] Running /tmp/dist-test-tasko3a_Hu/build/tsan/bin/kudu
16:19:59.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasko3a_Hu/build/tsan/bin/kudu
16:19:59.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/ts-0/wal
16:19:59.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/ts-0/data
16:19:59.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:19:59.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:19:59.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:19:59.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:19:59.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:19:59.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:19:59.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/ts-0/logs
16:19:59.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/ts-0/data/info.pb
16:19:59.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:19:59.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:19:59.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:19:59.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:19:59.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:19:59.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:19:59.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:19:59.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:19:59.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
16:19:59.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:19:59.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.10.1:0
16:19:59.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.10.1
16:19:59.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.10.1
16:19:59.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:19:59.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.10.62:37357,127.22.10.61:38773,127.22.10.60:39017
16:19:59.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/krb5kdc/kudu.keytab
16:19:59.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.22.10.1
16:19:59.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:19:59.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
16:19:59.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
16:19:59.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
16:19:59.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.10.20:38887
16:19:59.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:19:59.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:19:59.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:19:59.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false with env {KRB5CCNAME=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
16:19:59.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:59.594372 22896 catalog_manager.cc:1308] Loaded cluster ID: 2b04f3c8aee34f46a1410a54279e8a40
16:19:59.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:59.595023 22896 catalog_manager.cc:1601] T 00000000000000000000000000000000 P 9a7438b488774217a1a7eb7c632b81f8: loading cluster ID for follower catalog manager: success
16:19:59.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:59.607692 22896 catalog_manager.cc:1623] T 00000000000000000000000000000000 P 9a7438b488774217a1a7eb7c632b81f8: acquiring CA information for follower catalog manager: success
16:19:59.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:59.616717 22896 catalog_manager.cc:1651] T 00000000000000000000000000000000 P 9a7438b488774217a1a7eb7c632b81f8: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
16:19:59.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:59.647181 22894 catalog_manager.cc:1308] Loaded cluster ID: 2b04f3c8aee34f46a1410a54279e8a40
16:19:59.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:59.648924 22894 catalog_manager.cc:1601] T 00000000000000000000000000000000 P 083466e5de4a4ff682941e6b4624d3a0: loading cluster ID for follower catalog manager: success
16:19:59.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:59.657114 22894 catalog_manager.cc:1623] T 00000000000000000000000000000000 P 083466e5de4a4ff682941e6b4624d3a0: acquiring CA information for follower catalog manager: success
16:19:59.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:19:59.671267 22894 catalog_manager.cc:1651] T 00000000000000000000000000000000 P 083466e5de4a4ff682941e6b4624d3a0: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
16:19:59.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:59.944824 22916 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
16:19:59.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:59.945461 22916 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:19:59.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:19:59.946393 22916 flags.cc:432] Enabled unsafe flag: --never_fsync=true
16:20:00.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:20:00.014430 22916 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
16:20:00.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:20:00.015998 22916 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.22.10.1
16:20:00.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:00.095817 22916 tablet_server_runner.cc:78] Tablet server non-default flags:
16:20:00.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/ts-0/logs
16:20:00.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:20:00.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:20:00.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:20:00.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.10.20:38887
16:20:00.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:20:00.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:20:00.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/ts-0/data
16:20:00.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/ts-0/wal
16:20:00.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/krb5kdc/kudu.keytab
16:20:00.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.22.10.1
16:20:00.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:20:00.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:20:00.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.10.1:0
16:20:00.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:20:00.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:20:00.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
16:20:00.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
16:20:00.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:20:00.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:20:00.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/ts-0/data/info.pb
16:20:00.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.10.1
16:20:00.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:20:00.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.10.62:37357,127.22.10.61:38773,127.22.10.60:39017
16:20:00.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:20:00.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:20:00.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:20:00.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:20:00.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:20:00.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.10.1
16:20:00.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:20:00.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
16:20:00.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
16:20:00.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision faa920b8364ca43fd9d419253dab85c46533dc98
16:20:00.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:20:00.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 23 Sep 2026 15:47:26 UTC on e7f111948823
16:20:00.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 13962
16:20:00.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:20:00.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:00.097523 22916 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:20:00.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:00.098879 22916 file_cache.cc:504] Constructed file cache file cache with capacity 419430
16:20:00.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:20:00.138599 22922 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:20:01.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:20:00.161588 22926 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:20:01.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:00.159217 22916 server_base.cc:1061] running on GCE node
16:20:01.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:20:00.148277 22923 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:20:01.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:01.992173 22916 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:20:01.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:01.999363 22916 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:20:02.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:02.001487 22916 hybrid_clock.cc:648] HybridClock initialized: now 1790180402001440 us; error 52 us; skew 500 ppm
16:20:02.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:02.002710 22916 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:20:02.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 23 16:20:02 dist-test-slave-fqp8 krb5kdc[22580](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790180402, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.10.1@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:20:02.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:02.011029 22916 init.cc:377] Logged in from keytab as kudu/127.22.10.1@KRBTEST.COM (short username kudu)
16:20:02.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:02.018960 22916 webserver.cc:533] Webserver started at http://127.22.10.1:40159/ using document root <none> and password file <none>
16:20:02.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:02.021209 22916 fs_manager.cc:362] Metadata directory not provided
16:20:02.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:02.022472 22916 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:20:02.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:02.023658 22916 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
16:20:02.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:02.031533 22916 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/ts-0/data/instance:
16:20:02.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5d67bcf2ae6d4b17a2b0b922267d85ce"
16:20:02.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-23 16:20:02 on dist-test-slave-fqp8"
16:20:02.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:02.033620 22916 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/ts-0/wal/instance:
16:20:02.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5d67bcf2ae6d4b17a2b0b922267d85ce"
16:20:02.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-23 16:20:02 on dist-test-slave-fqp8"
16:20:02.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:02.045843 22916 fs_manager.cc:696] Time spent creating directory manager: real 0.011s user 0.010s sys 0.000s
16:20:02.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:02.055212 22936 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:20:02.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:02.056620 22916 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.002s sys 0.002s
16:20:02.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:02.057001 22916 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/ts-0/data,/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/ts-0/wal
16:20:02.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5d67bcf2ae6d4b17a2b0b922267d85ce"
16:20:02.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-23 16:20:02 on dist-test-slave-fqp8"
16:20:02.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:02.057404 22916 fs_report.cc:389] FS layout report
16:20:02.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:20:02.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/ts-0/wal
16:20:02.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/ts-0/wal
16:20:02.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/ts-0/data/data
16:20:02.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:20:02.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:20:02.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:20:02.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:20:02.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:20:02.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:20:02.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:20:02.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:20:02.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:20:02.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:20:02.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:20:02.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:02.123468 22916 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
16:20:02.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:02.133637 22916 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:20:02.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:02.134228 22916 kserver.cc:163] Server-wide thread pool size limit: 3276
16:20:02.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:02.137634 22916 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
16:20:02.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:02.144202 22916 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
16:20:02.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:02.144425 22916 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
16:20:02.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:02.144650 22916 ts_tablet_manager.cc:616] Registered 0 tablets
16:20:02.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:02.144798 22916 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
16:20:02.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:02.339710 22916 rpc_server.cc:307] RPC server started. Bound to: 127.22.10.1:36041
16:20:02.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:02.340029 23048 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.22.10.1:36041 every 8 connection(s)
16:20:02.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:02.342634 22916 server_base.cc:1193] Dumped server information to /tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/ts-0/data/info.pb
16:20:02.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:02.346822 22568 external_mini_cluster.cc:1434] Started /tmp/dist-test-tasko3a_Hu/build/tsan/bin/kudu as pid 22916
16:20:02.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:02.347316 22568 external_mini_cluster.cc:1448] Reading /tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/ts-0/wal/instance
16:20:02.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 23 16:20:02 dist-test-slave-fqp8 krb5kdc[22580](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790180402, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.10.1@KRBTEST.COM for kudu/127.22.10.61@KRBTEST.COM
16:20:02.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.22.10.2@KRBTEST.COM; defaulting to no policy
16:20:02.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:20:02.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.22.10.2@KRBTEST.COM" created.
16:20:02.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 23 16:20:02 dist-test-slave-fqp8 krb5kdc[22580](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790180402, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.10.1@KRBTEST.COM for kudu/127.22.10.62@KRBTEST.COM
16:20:02.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 23 16:20:02 dist-test-slave-fqp8 krb5kdc[22580](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790180402, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.10.1@KRBTEST.COM for kudu/127.22.10.60@KRBTEST.COM
16:20:02.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:02.411667 23049 heartbeater.cc:344] Connected to a master server at 127.22.10.61:38773
16:20:02.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:02.412209 23049 heartbeater.cc:461] Registering TS with master...
16:20:02.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:02.413986 23049 heartbeater.cc:507] Master 127.22.10.61:38773 requested a full tablet report, sending...
16:20:02.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:02.417886 22720 ts_manager.cc:194] Registered new tserver with Master: 5d67bcf2ae6d4b17a2b0b922267d85ce (127.22.10.1:36041)
16:20:02.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:02.421480 23051 heartbeater.cc:344] Connected to a master server at 127.22.10.62:37357
16:20:02.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:02.421917 23051 heartbeater.cc:461] Registering TS with master...
16:20:02.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:02.421967 23050 heartbeater.cc:344] Connected to a master server at 127.22.10.60:39017
16:20:02.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:02.422402 23050 heartbeater.cc:461] Registering TS with master...
16:20:02.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:02.423025 23051 heartbeater.cc:507] Master 127.22.10.62:37357 requested a full tablet report, sending...
16:20:02.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:02.423341 23050 heartbeater.cc:507] Master 127.22.10.60:39017 requested a full tablet report, sending...
16:20:02.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:02.426076 22808 ts_manager.cc:194] Registered new tserver with Master: 5d67bcf2ae6d4b17a2b0b922267d85ce (127.22.10.1:36041)
16:20:02.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:02.426203 22632 ts_manager.cc:194] Registered new tserver with Master: 5d67bcf2ae6d4b17a2b0b922267d85ce (127.22.10.1:36041)
16:20:02.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:20:02.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.22.10.2 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/krb5kdc/kudu.keytab.
16:20:02.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.22.10.2 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/krb5kdc/kudu.keytab.
16:20:02.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:02.429082 22808 master_service.cc:502] Signed X509 certificate for tserver {username='kudu', principal='kudu/127.22.10.1@KRBTEST.COM'} at 127.22.10.1:44087
16:20:02.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.22.10.2@KRBTEST.COM; defaulting to no policy
16:20:02.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:20:02.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.22.10.2@KRBTEST.COM" created.
16:20:02.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:20:02.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.22.10.2 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/krb5kdc/kudu.keytab.
16:20:02.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.22.10.2 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/krb5kdc/kudu.keytab.
16:20:02.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:02.478729 22568 external_mini_cluster.cc:1372] Running /tmp/dist-test-tasko3a_Hu/build/tsan/bin/kudu
16:20:02.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasko3a_Hu/build/tsan/bin/kudu
16:20:02.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/ts-1/wal
16:20:02.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/ts-1/data
16:20:02.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:20:02.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:20:02.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:20:02.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:20:02.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:20:02.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:20:02.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/ts-1/logs
16:20:02.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/ts-1/data/info.pb
16:20:02.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:20:02.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:20:02.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:20:02.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:20:02.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:20:02.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:20:02.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:20:02.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:20:02.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
16:20:02.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:20:02.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.10.2:0
16:20:02.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.10.2
16:20:02.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.10.2
16:20:02.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:20:02.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.10.62:37357,127.22.10.61:38773,127.22.10.60:39017
16:20:02.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/krb5kdc/kudu.keytab
16:20:02.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.22.10.2
16:20:02.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:20:02.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
16:20:02.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
16:20:02.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
16:20:02.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.10.20:38887
16:20:02.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:20:02.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:20:02.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:20:02.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false with env {KRB5CCNAME=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
16:20:02.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:20:02.839020 23063 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
16:20:02.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:20:02.839639 23063 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:20:02.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:20:02.840770 23063 flags.cc:432] Enabled unsafe flag: --never_fsync=true
16:20:02.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:20:02.877048 23063 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
16:20:02.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:20:02.878213 23063 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.22.10.2
16:20:02.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:02.917320 23063 tablet_server_runner.cc:78] Tablet server non-default flags:
16:20:02.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/ts-1/logs
16:20:02.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:20:02.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:20:02.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:20:02.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.10.20:38887
16:20:02.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:20:02.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:20:02.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/ts-1/data
16:20:02.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/ts-1/wal
16:20:02.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/krb5kdc/kudu.keytab
16:20:02.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.22.10.2
16:20:02.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:20:02.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:20:02.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.10.2:0
16:20:02.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:20:02.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:20:02.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
16:20:02.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
16:20:02.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:20:02.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:20:02.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/ts-1/data/info.pb
16:20:02.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.10.2
16:20:02.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:20:02.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.10.62:37357,127.22.10.61:38773,127.22.10.60:39017
16:20:02.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:20:02.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:20:02.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:20:02.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:20:02.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:20:02.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.10.2
16:20:02.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:20:02.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
16:20:02.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
16:20:02.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision faa920b8364ca43fd9d419253dab85c46533dc98
16:20:02.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:20:02.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 23 Sep 2026 15:47:26 UTC on e7f111948823
16:20:02.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 13962
16:20:02.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:20:02.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:02.918637 23063 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:20:02.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:02.919842 23063 file_cache.cc:504] Constructed file cache file cache with capacity 419430
16:20:02.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:20:02.933912 23069 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:20:03.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 23 16:20:03 dist-test-slave-fqp8 krb5kdc[22580](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790180403, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.10.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:20:03.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:03.395493 22610 init.cc:295] Successfully reacquired a new kerberos TGT
16:20:03.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:03.441490 23050 heartbeater.cc:499] Master 127.22.10.60:39017 was elected leader, sending a full tablet report...
16:20:04.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:20:04.336179 23068 debug-util.cc:398] Leaking SignalData structure 0x7b08000373a0 after lost signal to thread 23063
16:20:04.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:20:04.367798 23068 kernel_stack_watchdog.cc:198] Thread 23063 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 398ms:
16:20:04.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
16:20:04.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
16:20:04.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
16:20:04.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
16:20:04.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:20:02.935470 23070 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:20:04.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:20:04.373229 23063 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.438s user 0.001s sys 0.006s
16:20:04.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:20:04.373757 23071 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1430 milliseconds
16:20:04.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:20:04.373858 23063 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.438s user 0.001s sys 0.006s
16:20:04.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:20:04.375644 23073 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:20:04.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:04.375697 23063 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
16:20:04.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:04.377190 23063 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:20:04.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:04.381678 23063 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:20:04.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:04.383131 23063 hybrid_clock.cc:648] HybridClock initialized: now 1790180404383101 us; error 39 us; skew 500 ppm
16:20:04.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:04.383948 23063 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:20:04.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 23 16:20:04 dist-test-slave-fqp8 krb5kdc[22580](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790180404, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.10.2@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:20:04.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:04.389912 23063 init.cc:377] Logged in from keytab as kudu/127.22.10.2@KRBTEST.COM (short username kudu)
16:20:04.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:04.394966 23063 webserver.cc:533] Webserver started at http://127.22.10.2:37853/ using document root <none> and password file <none>
16:20:04.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:04.395815 23063 fs_manager.cc:362] Metadata directory not provided
16:20:04.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:04.395993 23063 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:20:04.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:04.396369 23063 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
16:20:04.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:04.400563 23063 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/ts-1/data/instance:
16:20:04.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0b6c620664ba45898067868764a3232e"
16:20:04.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-23 16:20:04 on dist-test-slave-fqp8"
16:20:04.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:04.401693 23063 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/ts-1/wal/instance:
16:20:04.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0b6c620664ba45898067868764a3232e"
16:20:04.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-23 16:20:04 on dist-test-slave-fqp8"
16:20:04.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:04.409009 23063 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.006s sys 0.000s
16:20:04.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:04.414558 23081 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:20:04.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:04.415596 23063 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.003s sys 0.002s
16:20:04.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:04.415946 23063 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/ts-1/data,/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/ts-1/wal
16:20:04.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0b6c620664ba45898067868764a3232e"
16:20:04.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-23 16:20:04 on dist-test-slave-fqp8"
16:20:04.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:04.416299 23063 fs_report.cc:389] FS layout report
16:20:04.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:20:04.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/ts-1/wal
16:20:04.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/ts-1/wal
16:20:04.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/ts-1/data/data
16:20:04.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:20:04.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:20:04.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:20:04.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:20:04.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:20:04.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:20:04.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:20:04.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:20:04.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:20:04.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:20:04.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:20:04.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:04.496217 23063 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
16:20:04.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:04.502274 23063 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:20:04.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:04.502810 23063 kserver.cc:163] Server-wide thread pool size limit: 3276
16:20:04.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:04.505498 23063 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
16:20:04.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:04.509768 23063 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
16:20:04.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:04.510214 23063 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.001s
16:20:04.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:04.510541 23063 ts_tablet_manager.cc:616] Registered 0 tablets
16:20:04.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:04.510735 23063 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
16:20:04.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:04.693895 23063 rpc_server.cc:307] RPC server started. Bound to: 127.22.10.2:35797
16:20:04.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:04.694051 23193 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.22.10.2:35797 every 8 connection(s)
16:20:04.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:04.696593 23063 server_base.cc:1193] Dumped server information to /tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/ts-1/data/info.pb
16:20:04.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:04.704250 22568 external_mini_cluster.cc:1434] Started /tmp/dist-test-tasko3a_Hu/build/tsan/bin/kudu as pid 23063
16:20:04.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:04.704723 22568 external_mini_cluster.cc:1448] Reading /tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/ts-1/wal/instance
16:20:04.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 23 16:20:04 dist-test-slave-fqp8 krb5kdc[22580](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790180404, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.10.2@KRBTEST.COM for kudu/127.22.10.62@KRBTEST.COM
16:20:04.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 23 16:20:04 dist-test-slave-fqp8 krb5kdc[22580](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790180404, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.10.2@KRBTEST.COM for kudu/127.22.10.61@KRBTEST.COM
16:20:04.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:04.786943 23194 heartbeater.cc:344] Connected to a master server at 127.22.10.61:38773
16:20:04.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:04.787485 23194 heartbeater.cc:461] Registering TS with master...
16:20:04.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:04.788872 23194 heartbeater.cc:507] Master 127.22.10.61:38773 requested a full tablet report, sending...
16:20:04.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:04.789088 23196 heartbeater.cc:344] Connected to a master server at 127.22.10.62:37357
16:20:04.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:04.789446 23196 heartbeater.cc:461] Registering TS with master...
16:20:04.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 23 16:20:04 dist-test-slave-fqp8 krb5kdc[22580](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790180404, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.10.2@KRBTEST.COM for kudu/127.22.10.60@KRBTEST.COM
16:20:04.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:04.790421 23196 heartbeater.cc:507] Master 127.22.10.62:37357 requested a full tablet report, sending...
16:20:04.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:04.793035 22720 ts_manager.cc:194] Registered new tserver with Master: 0b6c620664ba45898067868764a3232e (127.22.10.2:35797)
16:20:04.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:04.793305 22632 ts_manager.cc:194] Registered new tserver with Master: 0b6c620664ba45898067868764a3232e (127.22.10.2:35797)
16:20:04.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.22.10.3@KRBTEST.COM; defaulting to no policy
16:20:04.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:04.806255 23195 heartbeater.cc:344] Connected to a master server at 127.22.10.60:39017
16:20:04.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:04.806528 23195 heartbeater.cc:461] Registering TS with master...
16:20:04.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:04.807315 23195 heartbeater.cc:507] Master 127.22.10.60:39017 requested a full tablet report, sending...
16:20:04.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:20:04.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.22.10.3@KRBTEST.COM" created.
16:20:04.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:04.809522 22808 ts_manager.cc:194] Registered new tserver with Master: 0b6c620664ba45898067868764a3232e (127.22.10.2:35797)
16:20:04.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:04.811206 22808 master_service.cc:502] Signed X509 certificate for tserver {username='kudu', principal='kudu/127.22.10.2@KRBTEST.COM'} at 127.22.10.2:56301
16:20:04.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:20:04.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.22.10.3 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/krb5kdc/kudu.keytab.
16:20:04.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.22.10.3 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/krb5kdc/kudu.keytab.
16:20:04.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.22.10.3@KRBTEST.COM; defaulting to no policy
16:20:04.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:20:04.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.22.10.3@KRBTEST.COM" created.
16:20:04.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:20:04.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.22.10.3 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/krb5kdc/kudu.keytab.
16:20:04.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.22.10.3 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/krb5kdc/kudu.keytab.
16:20:04.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:04.874640 22568 external_mini_cluster.cc:1372] Running /tmp/dist-test-tasko3a_Hu/build/tsan/bin/kudu
16:20:04.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasko3a_Hu/build/tsan/bin/kudu
16:20:04.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/ts-2/wal
16:20:04.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/ts-2/data
16:20:04.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:20:04.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:20:04.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:20:04.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:20:04.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:20:04.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:20:04.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/ts-2/logs
16:20:04.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/ts-2/data/info.pb
16:20:04.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:20:04.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:20:04.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:20:04.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:20:04.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:20:04.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:20:04.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:20:04.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:20:04.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
16:20:04.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:20:04.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.10.3:0
16:20:04.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.10.3
16:20:04.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.10.3
16:20:04.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:20:04.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.10.62:37357,127.22.10.61:38773,127.22.10.60:39017
16:20:04.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/krb5kdc/kudu.keytab
16:20:04.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.22.10.3
16:20:04.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:20:04.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
16:20:04.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
16:20:04.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
16:20:04.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.10.20:38887
16:20:04.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:20:04.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:20:04.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:20:04.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false with env {KRB5CCNAME=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
16:20:05.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:20:05.228106 23209 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
16:20:05.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:20:05.228578 23209 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:20:05.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:20:05.229204 23209 flags.cc:432] Enabled unsafe flag: --never_fsync=true
16:20:05.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:20:05.264778 23209 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
16:20:05.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:20:05.265772 23209 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.22.10.3
16:20:05.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:05.305286 23209 tablet_server_runner.cc:78] Tablet server non-default flags:
16:20:05.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/ts-2/logs
16:20:05.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:20:05.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:20:05.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:20:05.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.10.20:38887
16:20:05.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:20:05.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:20:05.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/ts-2/data
16:20:05.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/ts-2/wal
16:20:05.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/krb5kdc/kudu.keytab
16:20:05.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.22.10.3
16:20:05.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:20:05.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:20:05.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.10.3:0
16:20:05.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:20:05.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:20:05.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
16:20:05.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
16:20:05.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:20:05.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:20:05.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/ts-2/data/info.pb
16:20:05.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.10.3
16:20:05.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:20:05.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.10.62:37357,127.22.10.61:38773,127.22.10.60:39017
16:20:05.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:20:05.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:20:05.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:20:05.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:20:05.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:20:05.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.10.3
16:20:05.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:20:05.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
16:20:05.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
16:20:05.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision faa920b8364ca43fd9d419253dab85c46533dc98
16:20:05.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:20:05.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 23 Sep 2026 15:47:26 UTC on e7f111948823
16:20:05.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 13962
16:20:05.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:20:05.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:05.306800 23209 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:20:05.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:05.308036 23209 file_cache.cc:504] Constructed file cache file cache with capacity 419430
16:20:05.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:20:05.324595 23217 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:20:05.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 23 16:20:05 dist-test-slave-fqp8 krb5kdc[22580](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790180405, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.10.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:20:05.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:05.741256 22699 init.cc:295] Successfully reacquired a new kerberos TGT
16:20:05.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:05.815188 23195 heartbeater.cc:499] Master 127.22.10.60:39017 was elected leader, sending a full tablet report...
16:20:07.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:20:05.326867 23218 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:20:07.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:20:07.160271 23220 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:20:07.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:20:06.726818 23216 debug-util.cc:398] Leaking SignalData structure 0x7b08000373a0 after lost signal to thread 23209
16:20:07.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:20:07.157627 23209 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.832s user 0.655s sys 1.081s
16:20:07.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:20:07.161546 23209 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.836s user 0.655s sys 1.082s
16:20:07.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:20:07.165388 23219 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1834 milliseconds
16:20:07.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:07.165398 23209 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
16:20:07.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:07.166682 23209 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:20:07.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:07.168654 23209 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:20:07.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:07.169965 23209 hybrid_clock.cc:648] HybridClock initialized: now 1790180407169992 us; error 101 us; skew 500 ppm
16:20:07.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:07.170707 23209 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:20:07.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 23 16:20:07 dist-test-slave-fqp8 krb5kdc[22580](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790180407, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.10.3@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:20:07.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:07.177138 23209 init.cc:377] Logged in from keytab as kudu/127.22.10.3@KRBTEST.COM (short username kudu)
16:20:07.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:07.181649 23209 webserver.cc:533] Webserver started at http://127.22.10.3:34481/ using document root <none> and password file <none>
16:20:07.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:07.182626 23209 fs_manager.cc:362] Metadata directory not provided
16:20:07.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:07.182802 23209 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:20:07.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:07.183188 23209 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
16:20:07.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:07.187772 23209 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/ts-2/data/instance:
16:20:07.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9575fea75d744ebd8e37e6270a83ceb4"
16:20:07.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-23 16:20:07 on dist-test-slave-fqp8"
16:20:07.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:07.188768 23209 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/ts-2/wal/instance:
16:20:07.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9575fea75d744ebd8e37e6270a83ceb4"
16:20:07.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-23 16:20:07 on dist-test-slave-fqp8"
16:20:07.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:07.196048 23209 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.009s sys 0.000s
16:20:07.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:07.201256 23231 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:20:07.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:07.202431 23209 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.001s sys 0.002s
16:20:07.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:07.202692 23209 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/ts-2/data,/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/ts-2/wal
16:20:07.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9575fea75d744ebd8e37e6270a83ceb4"
16:20:07.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-23 16:20:07 on dist-test-slave-fqp8"
16:20:07.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:07.202975 23209 fs_report.cc:389] FS layout report
16:20:07.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:20:07.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/ts-2/wal
16:20:07.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/ts-2/wal
16:20:07.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/ts-2/data/data
16:20:07.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:20:07.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:20:07.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:20:07.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:20:07.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:20:07.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:20:07.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:20:07.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:20:07.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:20:07.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:20:07.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:20:07.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:07.268092 23209 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
16:20:07.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:07.276661 23209 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:20:07.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:07.277070 23209 kserver.cc:163] Server-wide thread pool size limit: 3276
16:20:07.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:07.280053 23209 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
16:20:07.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:07.284057 23209 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
16:20:07.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:07.284241 23209 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
16:20:07.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:07.284425 23209 ts_tablet_manager.cc:616] Registered 0 tablets
16:20:07.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:07.284557 23209 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
16:20:07.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:07.442237 23209 rpc_server.cc:307] RPC server started. Bound to: 127.22.10.3:38899
16:20:07.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:07.442312 23343 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.22.10.3:38899 every 8 connection(s)
16:20:07.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:07.445676 23209 server_base.cc:1193] Dumped server information to /tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/ts-2/data/info.pb
16:20:07.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:07.456166 22568 external_mini_cluster.cc:1434] Started /tmp/dist-test-tasko3a_Hu/build/tsan/bin/kudu as pid 23209
16:20:07.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:07.456734 22568 external_mini_cluster.cc:1448] Reading /tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/ts-2/wal/instance
16:20:07.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 23 16:20:07 dist-test-slave-fqp8 krb5kdc[22580](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790180407, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.10.3@KRBTEST.COM for kudu/127.22.10.60@KRBTEST.COM
16:20:07.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 23 16:20:07 dist-test-slave-fqp8 krb5kdc[22580](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790180407, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.10.3@KRBTEST.COM for kudu/127.22.10.61@KRBTEST.COM
16:20:07.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 23 16:20:07 dist-test-slave-fqp8 krb5kdc[22580](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790180407, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.10.3@KRBTEST.COM for kudu/127.22.10.62@KRBTEST.COM
16:20:07.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:07.516746 23345 heartbeater.cc:344] Connected to a master server at 127.22.10.60:39017
16:20:07.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:07.517093 23344 heartbeater.cc:344] Connected to a master server at 127.22.10.61:38773
16:20:07.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:07.517349 23345 heartbeater.cc:461] Registering TS with master...
16:20:07.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:07.517500 23344 heartbeater.cc:461] Registering TS with master...
16:20:07.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:07.518918 23345 heartbeater.cc:507] Master 127.22.10.60:39017 requested a full tablet report, sending...
16:20:07.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:07.520818 23344 heartbeater.cc:507] Master 127.22.10.61:38773 requested a full tablet report, sending...
16:20:07.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:07.523480 22807 ts_manager.cc:194] Registered new tserver with Master: 9575fea75d744ebd8e37e6270a83ceb4 (127.22.10.3:38899)
16:20:07.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:07.524262 22720 ts_manager.cc:194] Registered new tserver with Master: 9575fea75d744ebd8e37e6270a83ceb4 (127.22.10.3:38899)
16:20:07.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:07.525532 22807 master_service.cc:502] Signed X509 certificate for tserver {username='kudu', principal='kudu/127.22.10.3@KRBTEST.COM'} at 127.22.10.3:38155
16:20:07.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:07.525893 23346 heartbeater.cc:344] Connected to a master server at 127.22.10.62:37357
16:20:07.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:07.526306 23346 heartbeater.cc:461] Registering TS with master...
16:20:07.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:07.527498 23346 heartbeater.cc:507] Master 127.22.10.62:37357 requested a full tablet report, sending...
16:20:07.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:07.530597 22632 ts_manager.cc:194] Registered new tserver with Master: 9575fea75d744ebd8e37e6270a83ceb4 (127.22.10.3:38899)
16:20:07.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:07.547736 22568 external_mini_cluster.cc:955] 3 TS(s) registered with all masters
16:20:07.562 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
16:20:07.563 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
16:20:07.572 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
16:20:07.573 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
16:20:07.576 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.22.10.62"
port: 37357
}
bound_http_address {
host: "127.22.10.62"
port: 33841
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.22.10.61"
port: 38773
}
bound_http_address {
host: "127.22.10.61"
port: 39977
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.22.10.60"
port: 39017
}
bound_http_address {
host: "127.22.10.60"
port: 35981
}
}
}
16:20:07.576 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
16:20:07.578 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.22.10.1"
port: 36041
}
bound_http_address {
host: "127.22.10.1"
port: 40159
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.22.10.2"
port: 35797
}
bound_http_address {
host: "127.22.10.2"
port: 37853
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.22.10.3"
port: 38899
}
bound_http_address {
host: "127.22.10.3"
port: 34481
}
}
}
16:20:07.578 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
16:20:07.579 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
16:20:07.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 23 16:20:07 dist-test-slave-fqp8 krb5kdc[22580](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790180407, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:20:07.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
16:20:07.630 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
16:20:07.633 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/krb5kdc/krb5cc
16:20:07.647 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
16:20:07.648 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/krb5kdc/krb5cc
16:20:07.650 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
16:20:07.683 [DEBUG - Time-limited test] (SecurityContext.java:158) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
16:20:07.685 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
16:20:07.686 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
16:20:07.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 23 16:20:07 dist-test-slave-fqp8 krb5kdc[22580](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790180407, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:20:07.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
16:20:07.748 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
16:20:07.748 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/krb5kdc/krb5cc
16:20:07.750 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
16:20:07.750 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.22.10.61:38773
16:20:07.751 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
16:20:07.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:07.752537 22568 external_mini_cluster.cc:1664] Killing /tmp/dist-test-tasko3a_Hu/build/tsan/bin/kudu with pid 22677
16:20:07.790 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
16:20:07.790 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.22.10.60:39017
16:20:07.790 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
16:20:07.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:07.791297 22568 external_mini_cluster.cc:1664] Killing /tmp/dist-test-tasko3a_Hu/build/tsan/bin/kudu with pid 22768
16:20:07.821 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
16:20:07.821 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.22.10.62:37357
16:20:07.822 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
16:20:07.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:07.822610 22568 external_mini_cluster.cc:1664] Killing /tmp/dist-test-tasko3a_Hu/build/tsan/bin/kudu with pid 22596
16:20:07.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:20:07.826157 23194 heartbeater.cc:646] Failed to heartbeat to 127.22.10.61:38773 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.22.10.61:38773: connect: Connection refused (error 111)
16:20:07.856 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
16:20:07.856 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.22.10.61:38773
16:20:07.860 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
16:20:07.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:07.862543 22568 external_mini_cluster.cc:1372] Running /tmp/dist-test-tasko3a_Hu/build/tsan/bin/kudu
16:20:07.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasko3a_Hu/build/tsan/bin/kudu
16:20:07.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/master-1/wal
16:20:07.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/master-1/data
16:20:07.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:20:07.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:20:07.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:20:07.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:20:07.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:20:07.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:20:07.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/master-1/logs
16:20:07.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/master-1/data/info.pb
16:20:07.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:20:07.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:20:07.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:20:07.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:20:07.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:20:07.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:20:07.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:20:07.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:20:07.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
16:20:07.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:20:07.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:20:07.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:20:07.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.10.61:38773
16:20:07.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.10.61
16:20:07.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39977
16:20:07.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/krb5kdc/kudu.keytab
16:20:07.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.22.10.61
16:20:07.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:20:07.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
16:20:07.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
16:20:07.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
16:20:07.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.10.20:38887
16:20:07.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:20:07.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:20:07.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:20:07.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:20:07.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.10.62:37357,127.22.10.61:38773,127.22.10.60:39017
16:20:07.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:20:07.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41371/jwks.json
16:20:07.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
16:20:07.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
16:20:08.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:20:08.189383 23359 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
16:20:08.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:20:08.190156 23359 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:20:08.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:20:08.190974 23359 flags.cc:432] Enabled unsafe flag: --never_fsync=true
16:20:08.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:20:08.228205 23359 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
16:20:08.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:20:08.228507 23359 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
16:20:08.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:20:08.228945 23359 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
16:20:08.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:20:08.229300 23359 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
16:20:08.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:20:08.229677 23359 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
16:20:08.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:20:08.230103 23359 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
16:20:08.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:20:08.230357 23359 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:41371/jwks.json
16:20:08.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:08.270799 23359 master_runner.cc:387] Master server non-default flags:
16:20:08.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/master-1/logs
16:20:08.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:20:08.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:20:08.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:20:08.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.10.20:38887
16:20:08.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:20:08.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:20:08.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/master-1/data
16:20:08.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/master-1/wal
16:20:08.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
16:20:08.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:20:08.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.10.62:37357,127.22.10.61:38773,127.22.10.60:39017
16:20:08.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/krb5kdc/kudu.keytab
16:20:08.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.22.10.61
16:20:08.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:20:08.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:20:08.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:20:08.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.10.61:38773
16:20:08.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:20:08.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:20:08.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:20:08.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41371/jwks.json
16:20:08.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:20:08.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
16:20:08.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
16:20:08.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:20:08.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:20:08.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/master-1/data/info.pb
16:20:08.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.10.61
16:20:08.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39977
16:20:08.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:20:08.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:20:08.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:20:08.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:20:08.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:20:08.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:20:08.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
16:20:08.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
16:20:08.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision faa920b8364ca43fd9d419253dab85c46533dc98
16:20:08.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:20:08.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 23 Sep 2026 15:47:26 UTC on e7f111948823
16:20:08.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 13962
16:20:08.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:20:08.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:08.272548 23359 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:20:08.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:08.274128 23359 file_cache.cc:504] Constructed file cache file cache with capacity 419430
16:20:08.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:20:08.285169 23365 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:20:08.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:20:08.492260 23051 heartbeater.cc:646] Failed to heartbeat to 127.22.10.62:37357 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.22.10.62:37357: connect: Connection refused (error 111)
16:20:08.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:08.529978 23345 heartbeater.cc:499] Master 127.22.10.60:39017 was elected leader, sending a full tablet report...
16:20:08.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:20:08.538853 23346 heartbeater.cc:646] Failed to heartbeat to 127.22.10.62:37357 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.22.10.62:37357: connect: Connection refused (error 111)
16:20:09.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:20:08.286968 23366 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:20:09.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:20:09.657106 23368 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:20:09.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:20:09.659653 23367 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1369 milliseconds
16:20:09.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:09.659754 23359 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
16:20:09.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:09.661123 23359 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:20:09.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:09.663857 23359 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:20:09.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:09.665251 23359 hybrid_clock.cc:648] HybridClock initialized: now 1790180409665209 us; error 39 us; skew 500 ppm
16:20:09.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:09.666190 23359 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:20:09.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 23 16:20:09 dist-test-slave-fqp8 krb5kdc[22580](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790180409, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.10.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:20:09.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:09.672806 23359 init.cc:377] Logged in from keytab as kudu/127.22.10.61@KRBTEST.COM (short username kudu)
16:20:09.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:09.678257 23359 webserver.cc:533] Webserver started at http://127.22.10.61:39977/ using document root <none> and password file <none>
16:20:09.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:09.679210 23359 fs_manager.cc:362] Metadata directory not provided
16:20:09.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:09.679472 23359 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:20:09.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:09.687974 23359 fs_manager.cc:714] Time spent opening directory manager: real 0.005s user 0.005s sys 0.000s
16:20:09.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:09.694180 23388 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:20:09.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:09.695487 23359 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.004s sys 0.000s
16:20:09.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:09.695845 23359 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/master-1/data,/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/master-1/wal
16:20:09.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9a7438b488774217a1a7eb7c632b81f8"
16:20:09.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-23 16:19:55 on dist-test-slave-fqp8"
16:20:09.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:09.698568 23359 fs_report.cc:389] FS layout report
16:20:09.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:20:09.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/master-1/wal
16:20:09.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/master-1/wal
16:20:09.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/master-1/data/data
16:20:09.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:20:09.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:20:09.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:20:09.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:20:09.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:20:09.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:20:09.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:20:09.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:20:09.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:20:09.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:20:09.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:20:09.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:09.782071 23359 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
16:20:09.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:09.793207 23359 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:20:09.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:09.793773 23359 kserver.cc:163] Server-wide thread pool size limit: 3276
16:20:09.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:09.877480 23359 rpc_server.cc:307] RPC server started. Bound to: 127.22.10.61:38773
16:20:09.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:09.877596 23441 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.22.10.61:38773 every 8 connection(s)
16:20:09.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:09.881042 23359 server_base.cc:1193] Dumped server information to /tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/master-1/data/info.pb
16:20:09.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:09.883164 22568 external_mini_cluster.cc:1434] Started /tmp/dist-test-tasko3a_Hu/build/tsan/bin/kudu as pid 23359
16:20:09.884 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
16:20:09.884 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.22.10.60:39017
16:20:09.884 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
16:20:09.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:09.886271 22568 external_mini_cluster.cc:1372] Running /tmp/dist-test-tasko3a_Hu/build/tsan/bin/kudu
16:20:09.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasko3a_Hu/build/tsan/bin/kudu
16:20:09.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/master-2/wal
16:20:09.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/master-2/data
16:20:09.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:20:09.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:20:09.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:20:09.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:20:09.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:20:09.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:20:09.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/master-2/logs
16:20:09.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/master-2/data/info.pb
16:20:09.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:20:09.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:20:09.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:20:09.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:20:09.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:20:09.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:20:09.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:20:09.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:20:09.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
16:20:09.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:20:09.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:20:09.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:20:09.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.10.60:39017
16:20:09.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.10.60
16:20:09.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35981
16:20:09.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/krb5kdc/kudu.keytab
16:20:09.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.22.10.60
16:20:09.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:20:09.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
16:20:09.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
16:20:09.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
16:20:09.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.10.20:38887
16:20:09.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:20:09.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:20:09.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:20:09.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:20:09.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.10.62:37357,127.22.10.61:38773,127.22.10.60:39017
16:20:09.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:20:09.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41371/jwks.json
16:20:09.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
16:20:09.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
16:20:09.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:09.897617 23442 sys_catalog.cc:263] Verifying existing consensus state
16:20:09.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:09.904116 23442 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9a7438b488774217a1a7eb7c632b81f8: Bootstrap starting.
16:20:09.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:09.949692 23442 log.cc:826] T 00000000000000000000000000000000 P 9a7438b488774217a1a7eb7c632b81f8: Log is configured to *not* fsync() on all Append() calls
16:20:09.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:09.966270 23442 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9a7438b488774217a1a7eb7c632b81f8: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
16:20:09.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:09.967212 23442 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9a7438b488774217a1a7eb7c632b81f8: Bootstrap complete.
16:20:09.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:09.993760 23442 raft_consensus.cc:359] T 00000000000000000000000000000000 P 9a7438b488774217a1a7eb7c632b81f8 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "083466e5de4a4ff682941e6b4624d3a0" member_type: VOTER last_known_addr { host: "127.22.10.62" port: 37357 } } peers { permanent_uuid: "9a7438b488774217a1a7eb7c632b81f8" member_type: VOTER last_known_addr { host: "127.22.10.61" port: 38773 } } peers { permanent_uuid: "fc4bbd8cf09e438895593b7651c63f06" member_type: VOTER last_known_addr { host: "127.22.10.60" port: 39017 } }
16:20:09.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:09.996517 23442 raft_consensus.cc:740] T 00000000000000000000000000000000 P 9a7438b488774217a1a7eb7c632b81f8 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9a7438b488774217a1a7eb7c632b81f8, State: Initialized, Role: FOLLOWER
16:20:09.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:09.997700 23442 consensus_queue.cc:260] T 00000000000000000000000000000000 P 9a7438b488774217a1a7eb7c632b81f8 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "083466e5de4a4ff682941e6b4624d3a0" member_type: VOTER last_known_addr { host: "127.22.10.62" port: 37357 } } peers { permanent_uuid: "9a7438b488774217a1a7eb7c632b81f8" member_type: VOTER last_known_addr { host: "127.22.10.61" port: 38773 } } peers { permanent_uuid: "fc4bbd8cf09e438895593b7651c63f06" member_type: VOTER last_known_addr { host: "127.22.10.60" port: 39017 } }
16:20:10.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:10.001264 23447 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9a7438b488774217a1a7eb7c632b81f8 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "083466e5de4a4ff682941e6b4624d3a0" member_type: VOTER last_known_addr { host: "127.22.10.62" port: 37357 } } peers { permanent_uuid: "9a7438b488774217a1a7eb7c632b81f8" member_type: VOTER last_known_addr { host: "127.22.10.61" port: 38773 } } peers { permanent_uuid: "fc4bbd8cf09e438895593b7651c63f06" member_type: VOTER last_known_addr { host: "127.22.10.60" port: 39017 } } }
16:20:10.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:10.002491 23447 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9a7438b488774217a1a7eb7c632b81f8 [sys.catalog]: This master's current role is: FOLLOWER
16:20:10.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:10.003307 23442 sys_catalog.cc:565] T 00000000000000000000000000000000 P 9a7438b488774217a1a7eb7c632b81f8 [sys.catalog]: configured and running, proceeding with master startup.
16:20:10.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:10.037927 23458 catalog_manager.cc:1308] Loaded cluster ID: 2b04f3c8aee34f46a1410a54279e8a40
16:20:10.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:10.038345 23458 catalog_manager.cc:1601] T 00000000000000000000000000000000 P 9a7438b488774217a1a7eb7c632b81f8: loading cluster ID for follower catalog manager: success
16:20:10.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:10.048552 23458 catalog_manager.cc:1623] T 00000000000000000000000000000000 P 9a7438b488774217a1a7eb7c632b81f8: acquiring CA information for follower catalog manager: success
16:20:10.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:10.057391 23458 catalog_manager.cc:1651] T 00000000000000000000000000000000 P 9a7438b488774217a1a7eb7c632b81f8: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
16:20:10.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:10.033684 23359 master_runner.cc:187] Error getting master registration for 127.22.10.62:37357: Network error: Client connection negotiation failed: client connection to 127.22.10.62:37357: connect: Connection refused (error 111),
16:20:10.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:10.061491 23359 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
16:20:10.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:20:10.259151 23444 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
16:20:10.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:20:10.259701 23444 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:20:10.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:20:10.260289 23444 flags.cc:432] Enabled unsafe flag: --never_fsync=true
16:20:10.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:20:10.295753 23444 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
16:20:10.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:20:10.295984 23444 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
16:20:10.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:20:10.296526 23444 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
16:20:10.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:20:10.296846 23444 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
16:20:10.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:20:10.297235 23444 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
16:20:10.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:20:10.297662 23444 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
16:20:10.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:20:10.297971 23444 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:41371/jwks.json
16:20:10.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:10.342654 23444 master_runner.cc:387] Master server non-default flags:
16:20:10.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/master-2/logs
16:20:10.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:20:10.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:20:10.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:20:10.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.10.20:38887
16:20:10.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:20:10.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:20:10.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/master-2/data
16:20:10.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/master-2/wal
16:20:10.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
16:20:10.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:20:10.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.10.62:37357,127.22.10.61:38773,127.22.10.60:39017
16:20:10.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/krb5kdc/kudu.keytab
16:20:10.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.22.10.60
16:20:10.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:20:10.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:20:10.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:20:10.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.10.60:39017
16:20:10.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:20:10.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:20:10.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:20:10.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41371/jwks.json
16:20:10.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:20:10.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
16:20:10.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
16:20:10.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:20:10.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:20:10.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/master-2/data/info.pb
16:20:10.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.10.60
16:20:10.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35981
16:20:10.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:20:10.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:20:10.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:20:10.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:20:10.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:20:10.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:20:10.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
16:20:10.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
16:20:10.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision faa920b8364ca43fd9d419253dab85c46533dc98
16:20:10.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:20:10.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 23 Sep 2026 15:47:26 UTC on e7f111948823
16:20:10.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 13962
16:20:10.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:20:10.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:10.344448 23444 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:20:10.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:10.346031 23444 file_cache.cc:504] Constructed file cache file cache with capacity 419430
16:20:10.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:20:10.360657 23467 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:20:10.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:20:10.384322 23340 debug-util.cc:398] Leaking SignalData structure 0x7b08000ae180 after lost signal to thread 23211
16:20:10.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:20:10.385874 23340 debug-util.cc:398] Leaking SignalData structure 0x7b08000b5dc0 after lost signal to thread 23343
16:20:10.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:10.631124 23049 heartbeater.cc:344] Connected to a master server at 127.22.10.61:38773
16:20:10.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:10.638170 23406 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "5d67bcf2ae6d4b17a2b0b922267d85ce" instance_seqno: 1790180402297763) as {username='kudu', principal='kudu/127.22.10.1@KRBTEST.COM'} at 127.22.10.1:59939; Asking this server to re-register.
16:20:10.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:10.639848 23049 heartbeater.cc:461] Registering TS with master...
16:20:10.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:10.643952 23406 ts_manager.cc:194] Registered new tserver with Master: 5d67bcf2ae6d4b17a2b0b922267d85ce (127.22.10.1:36041)
16:20:10.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:10.654500 23344 heartbeater.cc:344] Connected to a master server at 127.22.10.61:38773
16:20:10.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:10.657953 23406 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "9575fea75d744ebd8e37e6270a83ceb4" instance_seqno: 1790180407404652) as {username='kudu', principal='kudu/127.22.10.3@KRBTEST.COM'} at 127.22.10.3:59989; Asking this server to re-register.
16:20:10.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:10.659901 23344 heartbeater.cc:461] Registering TS with master...
16:20:10.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:10.664112 23406 ts_manager.cc:194] Registered new tserver with Master: 9575fea75d744ebd8e37e6270a83ceb4 (127.22.10.3:38899)
16:20:10.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:20:10.798627 22267 debug-util.cc:398] Leaking SignalData structure 0x7b08000addc0 after lost signal to thread 22130
16:20:10.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:20:10.800165 22267 debug-util.cc:398] Leaking SignalData structure 0x7b08000b6080 after lost signal to thread 22273
16:20:10.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:10.871541 23194 heartbeater.cc:344] Connected to a master server at 127.22.10.61:38773
16:20:10.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:10.875213 23406 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "0b6c620664ba45898067868764a3232e" instance_seqno: 1790180404651805) as {username='kudu', principal='kudu/127.22.10.2@KRBTEST.COM'} at 127.22.10.2:34983; Asking this server to re-register.
16:20:10.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:10.882961 23194 heartbeater.cc:461] Registering TS with master...
16:20:10.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:10.886600 23406 ts_manager.cc:194] Registered new tserver with Master: 0b6c620664ba45898067868764a3232e (127.22.10.2:35797)
16:20:11.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:20:11.567147 22552 debug-util.cc:398] Leaking SignalData structure 0x7b08000add40 after lost signal to thread 22427
16:20:11.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:20:11.568691 22552 debug-util.cc:398] Leaking SignalData structure 0x7b08000b8420 after lost signal to thread 22555
16:20:11.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:11.758693 23486 raft_consensus.cc:493] T 00000000000000000000000000000000 P 9a7438b488774217a1a7eb7c632b81f8 [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
16:20:11.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:11.759662 23486 raft_consensus.cc:515] T 00000000000000000000000000000000 P 9a7438b488774217a1a7eb7c632b81f8 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "083466e5de4a4ff682941e6b4624d3a0" member_type: VOTER last_known_addr { host: "127.22.10.62" port: 37357 } } peers { permanent_uuid: "9a7438b488774217a1a7eb7c632b81f8" member_type: VOTER last_known_addr { host: "127.22.10.61" port: 38773 } } peers { permanent_uuid: "fc4bbd8cf09e438895593b7651c63f06" member_type: VOTER last_known_addr { host: "127.22.10.60" port: 39017 } }
16:20:11.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:11.779353 23486 leader_election.cc:290] T 00000000000000000000000000000000 P 9a7438b488774217a1a7eb7c632b81f8 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 083466e5de4a4ff682941e6b4624d3a0 (127.22.10.62:37357), fc4bbd8cf09e438895593b7651c63f06 (127.22.10.60:39017)
16:20:11.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:20:11.796618 23390 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.22.10.60:39017: connect: Connection refused (error 111)
16:20:11.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:20:11.819766 23389 leader_election.cc:336] T 00000000000000000000000000000000 P 9a7438b488774217a1a7eb7c632b81f8 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 083466e5de4a4ff682941e6b4624d3a0 (127.22.10.62:37357): Network error: Client connection negotiation failed: client connection to 127.22.10.62:37357: connect: Connection refused (error 111)
16:20:11.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:20:11.830056 23390 leader_election.cc:336] T 00000000000000000000000000000000 P 9a7438b488774217a1a7eb7c632b81f8 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer fc4bbd8cf09e438895593b7651c63f06 (127.22.10.60:39017): Network error: Client connection negotiation failed: client connection to 127.22.10.60:39017: connect: Connection refused (error 111)
16:20:11.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:11.835168 23390 leader_election.cc:304] T 00000000000000000000000000000000 P 9a7438b488774217a1a7eb7c632b81f8 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 9a7438b488774217a1a7eb7c632b81f8; no voters: 083466e5de4a4ff682941e6b4624d3a0, fc4bbd8cf09e438895593b7651c63f06
16:20:11.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:11.846367 23486 raft_consensus.cc:2749] T 00000000000000000000000000000000 P 9a7438b488774217a1a7eb7c632b81f8 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
16:20:12.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 23 16:20:12 dist-test-slave-fqp8 krb5kdc[22580](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790180412, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.10.1@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:20:12.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:12.018711 22932 init.cc:295] Successfully reacquired a new kerberos TGT
16:20:12.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:20:11.766620 23466 debug-util.cc:398] Leaking SignalData structure 0x7b08000372a0 after lost signal to thread 23444
16:20:12.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:20:12.765986 23466 kernel_stack_watchdog.cc:198] Thread 23444 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 399ms:
16:20:12.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
16:20:12.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
16:20:12.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
16:20:12.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
16:20:12.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:20:12.766216 23444 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 2.405s user 0.626s sys 1.030s
16:20:12.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:20:12.766597 23444 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 2.406s user 0.626s sys 1.030s
16:20:12.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:20:10.362155 23468 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:20:12.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:20:12.769197 23470 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:20:12.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:20:12.773350 23469 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection timed out after 2412 milliseconds
16:20:12.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:12.773367 23444 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
16:20:12.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:12.775125 23444 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:20:12.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:12.788398 23444 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:20:12.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:12.791982 23444 hybrid_clock.cc:648] HybridClock initialized: now 1790180412791890 us; error 79 us; skew 500 ppm
16:20:12.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:12.793157 23444 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:20:12.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 23 16:20:12 dist-test-slave-fqp8 krb5kdc[22580](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790180412, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.10.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:20:12.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:12.805500 23444 init.cc:377] Logged in from keytab as kudu/127.22.10.60@KRBTEST.COM (short username kudu)
16:20:12.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:12.852293 23444 webserver.cc:533] Webserver started at http://127.22.10.60:35981/ using document root <none> and password file <none>
16:20:12.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:12.854171 23444 fs_manager.cc:362] Metadata directory not provided
16:20:12.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:12.854718 23444 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:20:12.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:12.872984 23444 fs_manager.cc:714] Time spent opening directory manager: real 0.013s user 0.008s sys 0.004s
16:20:12.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:12.892493 23505 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:20:12.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:12.894263 23444 fs_manager.cc:730] Time spent opening block manager: real 0.013s user 0.006s sys 0.000s
16:20:12.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:12.894861 23444 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/master-2/data,/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/master-2/wal
16:20:12.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fc4bbd8cf09e438895593b7651c63f06"
16:20:12.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-23 16:19:57 on dist-test-slave-fqp8"
16:20:12.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:12.898160 23444 fs_report.cc:389] FS layout report
16:20:12.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:20:12.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/master-2/wal
16:20:12.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/master-2/wal
16:20:12.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/master-2/data/data
16:20:12.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:20:12.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:20:12.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:20:12.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:20:12.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:20:12.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:20:12.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:20:12.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:20:12.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:20:12.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:20:12.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:20:13.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:13.024760 23444 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
16:20:13.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:13.041401 23444 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:20:13.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:13.047160 23444 kserver.cc:163] Server-wide thread pool size limit: 3276
16:20:13.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:13.529331 23557 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.22.10.60:39017 every 8 connection(s)
16:20:13.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:13.530030 23444 rpc_server.cc:307] RPC server started. Bound to: 127.22.10.60:39017
16:20:13.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:13.534898 23444 server_base.cc:1193] Dumped server information to /tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/master-2/data/info.pb
16:20:13.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:13.536687 22568 external_mini_cluster.cc:1434] Started /tmp/dist-test-tasko3a_Hu/build/tsan/bin/kudu as pid 23444
16:20:13.538 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
16:20:13.538 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.22.10.62:37357
16:20:13.539 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
16:20:13.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:13.540895 22568 external_mini_cluster.cc:1372] Running /tmp/dist-test-tasko3a_Hu/build/tsan/bin/kudu
16:20:13.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasko3a_Hu/build/tsan/bin/kudu
16:20:13.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/master-0/wal
16:20:13.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/master-0/data
16:20:13.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:20:13.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:20:13.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:20:13.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:20:13.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:20:13.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:20:13.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/master-0/logs
16:20:13.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/master-0/data/info.pb
16:20:13.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:20:13.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:20:13.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:20:13.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:20:13.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:20:13.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:20:13.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:20:13.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:20:13.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
16:20:13.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:20:13.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:20:13.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:20:13.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.10.62:37357
16:20:13.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.10.62
16:20:13.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33841
16:20:13.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/krb5kdc/kudu.keytab
16:20:13.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.22.10.62
16:20:13.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:20:13.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
16:20:13.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
16:20:13.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
16:20:13.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.10.20:38887
16:20:13.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:20:13.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:20:13.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:20:13.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:20:13.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.10.62:37357,127.22.10.61:38773,127.22.10.60:39017
16:20:13.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:20:13.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41371/jwks.json
16:20:13.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
16:20:13.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
16:20:13.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:13.588586 23558 sys_catalog.cc:263] Verifying existing consensus state
16:20:13.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:13.655516 23558 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P fc4bbd8cf09e438895593b7651c63f06: Bootstrap starting.
16:20:13.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 23 16:20:13 dist-test-slave-fqp8 krb5kdc[22580](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790180412, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.10.1@KRBTEST.COM for kudu/127.22.10.60@KRBTEST.COM
16:20:13.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:13.772121 23567 raft_consensus.cc:493] T 00000000000000000000000000000000 P 9a7438b488774217a1a7eb7c632b81f8 [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
16:20:13.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:13.776571 23567 raft_consensus.cc:515] T 00000000000000000000000000000000 P 9a7438b488774217a1a7eb7c632b81f8 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "083466e5de4a4ff682941e6b4624d3a0" member_type: VOTER last_known_addr { host: "127.22.10.62" port: 37357 } } peers { permanent_uuid: "9a7438b488774217a1a7eb7c632b81f8" member_type: VOTER last_known_addr { host: "127.22.10.61" port: 38773 } } peers { permanent_uuid: "fc4bbd8cf09e438895593b7651c63f06" member_type: VOTER last_known_addr { host: "127.22.10.60" port: 39017 } }
16:20:13.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:13.790599 23567 leader_election.cc:290] T 00000000000000000000000000000000 P 9a7438b488774217a1a7eb7c632b81f8 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 083466e5de4a4ff682941e6b4624d3a0 (127.22.10.62:37357), fc4bbd8cf09e438895593b7651c63f06 (127.22.10.60:39017)
16:20:13.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:20:13.819947 23389 leader_election.cc:336] T 00000000000000000000000000000000 P 9a7438b488774217a1a7eb7c632b81f8 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 083466e5de4a4ff682941e6b4624d3a0 (127.22.10.62:37357): Network error: Client connection negotiation failed: client connection to 127.22.10.62:37357: connect: Connection refused (error 111)
16:20:13.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:13.827904 23050 heartbeater.cc:344] Connected to a master server at 127.22.10.60:39017
16:20:13.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:13.843914 23345 heartbeater.cc:344] Connected to a master server at 127.22.10.60:39017
16:20:13.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:13.844415 23345 heartbeater.cc:499] Master 127.22.10.60:39017 was elected leader, sending a full tablet report...
16:20:13.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 23 16:20:13 dist-test-slave-fqp8 krb5kdc[22580](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790180409, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.10.61@KRBTEST.COM for kudu/127.22.10.60@KRBTEST.COM
16:20:14.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:20:14.020908 23570 negotiation.cc:336] Failed RPC negotiation. Trace:
16:20:14.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0923 16:20:13.894400 (+ 0us) reactor.cc:730] Submitting negotiation task for server connection from 127.22.10.2:48297 (local address 127.22.10.60:39017)
16:20:14.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0923 16:20:13.895130 (+ 730us) server_negotiation.cc:207] Beginning negotiation
16:20:14.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0923 16:20:13.895159 (+ 29us) server_negotiation.cc:400] Waiting for connection header
16:20:14.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0923 16:20:13.917387 (+ 22228us) server_negotiation.cc:408] Connection header received
16:20:14.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0923 16:20:13.929999 (+ 12612us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
16:20:14.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0923 16:20:13.930030 (+ 31us) server_negotiation.cc:462] Received NEGOTIATE request from client
16:20:14.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0923 16:20:13.930365 (+ 335us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
16:20:14.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0923 16:20:13.930523 (+ 158us) server_negotiation.cc:227] Negotiated authn=SASL
16:20:14.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0923 16:20:13.948108 (+ 17585us) server_negotiation.cc:366] Received TLS_HANDSHAKE NegotiatePB request
16:20:14.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0923 16:20:13.950079 (+ 1971us) server_negotiation.cc:378] Sending TLS_HANDSHAKE NegotiatePB response
16:20:14.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0923 16:20:13.965256 (+ 15177us) server_negotiation.cc:366] Received TLS_HANDSHAKE NegotiatePB request
16:20:14.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0923 16:20:13.965975 (+ 719us) server_negotiation.cc:658] Negotiated auth-only TLSv1.3 with cipher TLS_AES_128_GCM_SHA256 TLSv1.3 Kx=any Au=any Enc=AESGCM(128) Mac=AEAD
16:20:14.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0923 16:20:13.993334 (+ 27359us) server_negotiation.cc:366] Received SASL_INITIATE NegotiatePB request
16:20:14.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0923 16:20:13.993490 (+ 156us) server_negotiation.cc:882] Received SASL_INITIATE request from client
16:20:14.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0923 16:20:13.993526 (+ 36us) server_negotiation.cc:893] Client requested to use mechanism: GSSAPI
16:20:14.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0923 16:20:13.993678 (+ 152us) server_negotiation.cc:923] calling sasl_server_start()
16:20:14.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0923 16:20:14.001647 (+ 7969us) server_negotiation.cc:378] Sending SASL_CHALLENGE NegotiatePB response
16:20:14.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0923 16:20:14.018403 (+ 16756us) negotiation.cc:326] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.22.10.2:48297: BlockingRecv error: recv got EOF from 127.22.10.2:48297 (error 108)
16:20:14.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":125}
16:20:14.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:14.088666 23558 log.cc:826] T 00000000000000000000000000000000 P fc4bbd8cf09e438895593b7651c63f06: Log is configured to *not* fsync() on all Append() calls
16:20:14.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:14.176764 23558 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P fc4bbd8cf09e438895593b7651c63f06: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
16:20:14.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:14.180516 23558 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P fc4bbd8cf09e438895593b7651c63f06: Bootstrap complete.
16:20:14.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:20:14.255684 21907 debug-util.cc:398] Leaking SignalData structure 0x7b080007cae0 after lost signal to thread 21844
16:20:14.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:20:14.256421 21907 debug-util.cc:398] Leaking SignalData structure 0x7b08000a4460 after lost signal to thread 21910
16:20:14.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:14.311656 23558 raft_consensus.cc:359] T 00000000000000000000000000000000 P fc4bbd8cf09e438895593b7651c63f06 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "083466e5de4a4ff682941e6b4624d3a0" member_type: VOTER last_known_addr { host: "127.22.10.62" port: 37357 } } peers { permanent_uuid: "9a7438b488774217a1a7eb7c632b81f8" member_type: VOTER last_known_addr { host: "127.22.10.61" port: 38773 } } peers { permanent_uuid: "fc4bbd8cf09e438895593b7651c63f06" member_type: VOTER last_known_addr { host: "127.22.10.60" port: 39017 } }
16:20:14.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:14.325469 23558 raft_consensus.cc:740] T 00000000000000000000000000000000 P fc4bbd8cf09e438895593b7651c63f06 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: fc4bbd8cf09e438895593b7651c63f06, State: Initialized, Role: FOLLOWER
16:20:14.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:14.329555 23558 consensus_queue.cc:260] T 00000000000000000000000000000000 P fc4bbd8cf09e438895593b7651c63f06 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "083466e5de4a4ff682941e6b4624d3a0" member_type: VOTER last_known_addr { host: "127.22.10.62" port: 37357 } } peers { permanent_uuid: "9a7438b488774217a1a7eb7c632b81f8" member_type: VOTER last_known_addr { host: "127.22.10.61" port: 38773 } } peers { permanent_uuid: "fc4bbd8cf09e438895593b7651c63f06" member_type: VOTER last_known_addr { host: "127.22.10.60" port: 39017 } }
16:20:14.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:14.353317 23558 sys_catalog.cc:565] T 00000000000000000000000000000000 P fc4bbd8cf09e438895593b7651c63f06 [sys.catalog]: configured and running, proceeding with master startup.
16:20:14.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:14.353506 23532 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "9a7438b488774217a1a7eb7c632b81f8" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "fc4bbd8cf09e438895593b7651c63f06" is_pre_election: true
16:20:14.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:14.354491 23532 raft_consensus.cc:2468] T 00000000000000000000000000000000 P fc4bbd8cf09e438895593b7651c63f06 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 9a7438b488774217a1a7eb7c632b81f8 in term 1.
16:20:14.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:14.356048 23390 leader_election.cc:304] T 00000000000000000000000000000000 P 9a7438b488774217a1a7eb7c632b81f8 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 9a7438b488774217a1a7eb7c632b81f8, fc4bbd8cf09e438895593b7651c63f06; no voters: 083466e5de4a4ff682941e6b4624d3a0
16:20:14.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:14.352656 23575 sys_catalog.cc:455] T 00000000000000000000000000000000 P fc4bbd8cf09e438895593b7651c63f06 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "083466e5de4a4ff682941e6b4624d3a0" member_type: VOTER last_known_addr { host: "127.22.10.62" port: 37357 } } peers { permanent_uuid: "9a7438b488774217a1a7eb7c632b81f8" member_type: VOTER last_known_addr { host: "127.22.10.61" port: 38773 } } peers { permanent_uuid: "fc4bbd8cf09e438895593b7651c63f06" member_type: VOTER last_known_addr { host: "127.22.10.60" port: 39017 } } }
16:20:14.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:14.366611 23575 sys_catalog.cc:458] T 00000000000000000000000000000000 P fc4bbd8cf09e438895593b7651c63f06 [sys.catalog]: This master's current role is: FOLLOWER
16:20:14.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:14.368101 23576 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 9a7438b488774217a1a7eb7c632b81f8 [term 1 FOLLOWER]: Leader pre-election won for term 2
16:20:14.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:14.368692 23576 raft_consensus.cc:493] T 00000000000000000000000000000000 P 9a7438b488774217a1a7eb7c632b81f8 [term 1 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
16:20:14.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:14.369235 23576 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 9a7438b488774217a1a7eb7c632b81f8 [term 1 FOLLOWER]: Advancing to term 2
16:20:14.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 23 16:20:14 dist-test-slave-fqp8 krb5kdc[22580](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790180414, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.10.2@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:20:14.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:14.395110 23077 init.cc:295] Successfully reacquired a new kerberos TGT
16:20:14.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:14.395457 23576 raft_consensus.cc:515] T 00000000000000000000000000000000 P 9a7438b488774217a1a7eb7c632b81f8 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "083466e5de4a4ff682941e6b4624d3a0" member_type: VOTER last_known_addr { host: "127.22.10.62" port: 37357 } } peers { permanent_uuid: "9a7438b488774217a1a7eb7c632b81f8" member_type: VOTER last_known_addr { host: "127.22.10.61" port: 38773 } } peers { permanent_uuid: "fc4bbd8cf09e438895593b7651c63f06" member_type: VOTER last_known_addr { host: "127.22.10.60" port: 39017 } }
16:20:14.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:14.440192 23576 leader_election.cc:290] T 00000000000000000000000000000000 P 9a7438b488774217a1a7eb7c632b81f8 [CANDIDATE]: Term 2 election: Requested vote from peers 083466e5de4a4ff682941e6b4624d3a0 (127.22.10.62:37357), fc4bbd8cf09e438895593b7651c63f06 (127.22.10.60:39017)
16:20:14.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:14.441738 23532 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "9a7438b488774217a1a7eb7c632b81f8" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "fc4bbd8cf09e438895593b7651c63f06"
16:20:14.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:14.442379 23532 raft_consensus.cc:3060] T 00000000000000000000000000000000 P fc4bbd8cf09e438895593b7651c63f06 [term 1 FOLLOWER]: Advancing to term 2
16:20:14.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:14.452402 23532 raft_consensus.cc:2468] T 00000000000000000000000000000000 P fc4bbd8cf09e438895593b7651c63f06 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 9a7438b488774217a1a7eb7c632b81f8 in term 2.
16:20:14.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:14.453760 23390 leader_election.cc:304] T 00000000000000000000000000000000 P 9a7438b488774217a1a7eb7c632b81f8 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 9a7438b488774217a1a7eb7c632b81f8, fc4bbd8cf09e438895593b7651c63f06; no voters:
16:20:14.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:20:14.456682 23389 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.RequestConsensusVote -> {remote=127.22.10.62:37357, user_credentials={real_user=slave}} blocked reactor thread for 43637.9us
16:20:14.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:14.458081 23576 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 9a7438b488774217a1a7eb7c632b81f8 [term 2 FOLLOWER]: Leader election won for term 2
16:20:14.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:20:14.467489 23389 leader_election.cc:336] T 00000000000000000000000000000000 P 9a7438b488774217a1a7eb7c632b81f8 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 083466e5de4a4ff682941e6b4624d3a0 (127.22.10.62:37357): Network error: Client connection negotiation failed: client connection to 127.22.10.62:37357: connect: Connection refused (error 111)
16:20:14.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:14.474654 23576 raft_consensus.cc:697] T 00000000000000000000000000000000 P 9a7438b488774217a1a7eb7c632b81f8 [term 2 LEADER]: Becoming Leader. State: Replica: 9a7438b488774217a1a7eb7c632b81f8, State: Running, Role: LEADER
16:20:14.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:14.476250 23576 consensus_queue.cc:237] T 00000000000000000000000000000000 P 9a7438b488774217a1a7eb7c632b81f8 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "083466e5de4a4ff682941e6b4624d3a0" member_type: VOTER last_known_addr { host: "127.22.10.62" port: 37357 } } peers { permanent_uuid: "9a7438b488774217a1a7eb7c632b81f8" member_type: VOTER last_known_addr { host: "127.22.10.61" port: 38773 } } peers { permanent_uuid: "fc4bbd8cf09e438895593b7651c63f06" member_type: VOTER last_known_addr { host: "127.22.10.60" port: 39017 } }
16:20:14.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:14.503736 23584 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9a7438b488774217a1a7eb7c632b81f8 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 9a7438b488774217a1a7eb7c632b81f8. Latest consensus state: current_term: 2 leader_uuid: "9a7438b488774217a1a7eb7c632b81f8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "083466e5de4a4ff682941e6b4624d3a0" member_type: VOTER last_known_addr { host: "127.22.10.62" port: 37357 } } peers { permanent_uuid: "9a7438b488774217a1a7eb7c632b81f8" member_type: VOTER last_known_addr { host: "127.22.10.61" port: 38773 } } peers { permanent_uuid: "fc4bbd8cf09e438895593b7651c63f06" member_type: VOTER last_known_addr { host: "127.22.10.60" port: 39017 } } }
16:20:14.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:14.505064 23584 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9a7438b488774217a1a7eb7c632b81f8 [sys.catalog]: This master's current role is: LEADER
16:20:14.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:14.526450 23590 catalog_manager.cc:818] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
16:20:14.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:14.530891 23591 catalog_manager.cc:1524] Loading table and tablet metadata into memory...
16:20:14.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:14.569407 23591 catalog_manager.cc:1533] Initializing Kudu cluster ID...
16:20:14.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:14.612175 23591 catalog_manager.cc:1308] Loaded cluster ID: 2b04f3c8aee34f46a1410a54279e8a40
16:20:14.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:14.615593 23591 catalog_manager.cc:1544] Initializing Kudu internal certificate authority...
16:20:14.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:20:14.649994 23559 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
16:20:14.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:20:14.650779 23559 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:20:14.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:20:14.651841 23559 flags.cc:432] Enabled unsafe flag: --never_fsync=true
16:20:14.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:14.674532 23591 catalog_manager.cc:1553] Loading token signing keys...
16:20:14.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:14.559429 23444 master_runner.cc:187] Error getting master registration for 127.22.10.62:37357: Network error: Client connection negotiation failed: client connection to 127.22.10.62:37357: connect: Connection refused (error 111),
16:20:14.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:14.698777 23591 catalog_manager.cc:6230] T 00000000000000000000000000000000 P 9a7438b488774217a1a7eb7c632b81f8: Loaded TSK: 0
16:20:14.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:14.699576 23444 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
16:20:14.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:14.701020 23591 catalog_manager.cc:1563] Initializing in-progress tserver states...
16:20:14.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:20:14.792642 23559 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
16:20:14.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:20:14.792968 23559 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
16:20:14.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:20:14.793429 23559 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
16:20:14.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:20:14.793771 23559 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
16:20:14.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:20:14.794131 23559 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
16:20:14.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:20:14.794456 23559 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
16:20:14.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:20:14.794701 23559 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:41371/jwks.json
16:20:14.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:14.842525 23522 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "5d67bcf2ae6d4b17a2b0b922267d85ce" instance_seqno: 1790180402297763) as {username='kudu', principal='kudu/127.22.10.1@KRBTEST.COM'} at 127.22.10.1:40279; Asking this server to re-register.
16:20:14.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:14.844182 23050 heartbeater.cc:461] Registering TS with master...
16:20:14.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:14.847651 23522 ts_manager.cc:194] Registered new tserver with Master: 5d67bcf2ae6d4b17a2b0b922267d85ce (127.22.10.1:36041)
16:20:14.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:14.857578 23345 heartbeater.cc:499] Master 127.22.10.60:39017 was elected leader, sending a full tablet report...
16:20:14.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:14.860212 23522 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "9575fea75d744ebd8e37e6270a83ceb4" instance_seqno: 1790180407404652) as {username='kudu', principal='kudu/127.22.10.3@KRBTEST.COM'} at 127.22.10.3:52859; Asking this server to re-register.
16:20:14.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:14.862174 23345 heartbeater.cc:461] Registering TS with master...
16:20:14.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:14.865070 23522 ts_manager.cc:194] Registered new tserver with Master: 9575fea75d744ebd8e37e6270a83ceb4 (127.22.10.3:38899)
16:20:14.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:14.965289 23559 master_runner.cc:387] Master server non-default flags:
16:20:14.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/master-0/logs
16:20:14.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:20:14.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:20:14.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:20:14.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.10.20:38887
16:20:14.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:20:14.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:20:14.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/master-0/data
16:20:14.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/master-0/wal
16:20:14.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
16:20:14.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:20:14.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.10.62:37357,127.22.10.61:38773,127.22.10.60:39017
16:20:14.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/krb5kdc/kudu.keytab
16:20:14.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.22.10.62
16:20:14.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:20:14.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:20:14.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:20:14.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.10.62:37357
16:20:14.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:20:14.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:20:14.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:20:14.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41371/jwks.json
16:20:14.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:20:14.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
16:20:14.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
16:20:14.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:20:14.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:20:14.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/master-0/data/info.pb
16:20:14.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.10.62
16:20:14.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33841
16:20:14.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:20:14.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:20:14.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:20:14.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:20:14.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:20:14.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:20:14.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
16:20:14.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
16:20:14.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision faa920b8364ca43fd9d419253dab85c46533dc98
16:20:14.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:20:14.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 23 Sep 2026 15:47:26 UTC on e7f111948823
16:20:14.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 13962
16:20:14.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:20:14.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:14.976424 23559 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:20:14.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:14.978060 23559 file_cache.cc:504] Constructed file cache file cache with capacity 419430
16:20:14.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:20:14.992131 23389 consensus_peers.cc:597] T 00000000000000000000000000000000 P 9a7438b488774217a1a7eb7c632b81f8 -> Peer 083466e5de4a4ff682941e6b4624d3a0 (127.22.10.62:37357): Couldn't send request to peer 083466e5de4a4ff682941e6b4624d3a0. Status: Network error: Client connection negotiation failed: client connection to 127.22.10.62:37357: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
16:20:15.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:20:15.013929 23605 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:20:15.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 23 16:20:15 dist-test-slave-fqp8 krb5kdc[22580](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790180414, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.10.2@KRBTEST.COM for kudu/127.22.10.60@KRBTEST.COM
16:20:15.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:15.063620 23532 raft_consensus.cc:1275] T 00000000000000000000000000000000 P fc4bbd8cf09e438895593b7651c63f06 [term 2 FOLLOWER]: Refusing update from remote peer 9a7438b488774217a1a7eb7c632b81f8: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 2 index: 5. (index mismatch)
16:20:15.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:15.065856 23359 master_runner.cc:187] Error getting master registration for 127.22.10.62:37357: Network error: Client connection negotiation failed: client connection to 127.22.10.62:37357: connect: Connection refused (error 111),
16:20:15.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:15.066330 23359 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
16:20:15.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:15.066905 23584 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 9a7438b488774217a1a7eb7c632b81f8 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fc4bbd8cf09e438895593b7651c63f06" member_type: VOTER last_known_addr { host: "127.22.10.60" port: 39017 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
16:20:15.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:15.158649 23195 heartbeater.cc:344] Connected to a master server at 127.22.10.60:39017
16:20:15.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:15.188906 23522 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "0b6c620664ba45898067868764a3232e" instance_seqno: 1790180404651805) as {username='kudu', principal='kudu/127.22.10.2@KRBTEST.COM'} at 127.22.10.2:52563; Asking this server to re-register.
16:20:15.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:15.190689 23195 heartbeater.cc:461] Registering TS with master...
16:20:15.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:15.192636 23612 sys_catalog.cc:455] T 00000000000000000000000000000000 P fc4bbd8cf09e438895593b7651c63f06 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 9a7438b488774217a1a7eb7c632b81f8. Latest consensus state: current_term: 2 leader_uuid: "9a7438b488774217a1a7eb7c632b81f8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "083466e5de4a4ff682941e6b4624d3a0" member_type: VOTER last_known_addr { host: "127.22.10.62" port: 37357 } } peers { permanent_uuid: "9a7438b488774217a1a7eb7c632b81f8" member_type: VOTER last_known_addr { host: "127.22.10.61" port: 38773 } } peers { permanent_uuid: "fc4bbd8cf09e438895593b7651c63f06" member_type: VOTER last_known_addr { host: "127.22.10.60" port: 39017 } } }
16:20:15.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:15.204275 23612 sys_catalog.cc:458] T 00000000000000000000000000000000 P fc4bbd8cf09e438895593b7651c63f06 [sys.catalog]: This master's current role is: FOLLOWER
16:20:15.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:15.194892 23522 ts_manager.cc:194] Registered new tserver with Master: 0b6c620664ba45898067868764a3232e (127.22.10.2:35797)
16:20:15.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:15.255033 23612 sys_catalog.cc:455] T 00000000000000000000000000000000 P fc4bbd8cf09e438895593b7651c63f06 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "9a7438b488774217a1a7eb7c632b81f8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "083466e5de4a4ff682941e6b4624d3a0" member_type: VOTER last_known_addr { host: "127.22.10.62" port: 37357 } } peers { permanent_uuid: "9a7438b488774217a1a7eb7c632b81f8" member_type: VOTER last_known_addr { host: "127.22.10.61" port: 38773 } } peers { permanent_uuid: "fc4bbd8cf09e438895593b7651c63f06" member_type: VOTER last_known_addr { host: "127.22.10.60" port: 39017 } } }
16:20:15.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:15.256116 23612 sys_catalog.cc:458] T 00000000000000000000000000000000 P fc4bbd8cf09e438895593b7651c63f06 [sys.catalog]: This master's current role is: FOLLOWER
16:20:15.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:15.276958 23584 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9a7438b488774217a1a7eb7c632b81f8 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "9a7438b488774217a1a7eb7c632b81f8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "083466e5de4a4ff682941e6b4624d3a0" member_type: VOTER last_known_addr { host: "127.22.10.62" port: 37357 } } peers { permanent_uuid: "9a7438b488774217a1a7eb7c632b81f8" member_type: VOTER last_known_addr { host: "127.22.10.61" port: 38773 } } peers { permanent_uuid: "fc4bbd8cf09e438895593b7651c63f06" member_type: VOTER last_known_addr { host: "127.22.10.60" port: 39017 } } }
16:20:15.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:15.277678 23584 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9a7438b488774217a1a7eb7c632b81f8 [sys.catalog]: This master's current role is: LEADER
16:20:15.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:15.562832 23590 catalog_manager.cc:1308] Loaded cluster ID: 2b04f3c8aee34f46a1410a54279e8a40
16:20:15.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:15.564669 23590 catalog_manager.cc:1601] T 00000000000000000000000000000000 P fc4bbd8cf09e438895593b7651c63f06: loading cluster ID for follower catalog manager: success
16:20:15.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:15.584846 23590 catalog_manager.cc:1623] T 00000000000000000000000000000000 P fc4bbd8cf09e438895593b7651c63f06: acquiring CA information for follower catalog manager: success
16:20:15.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:15.591531 23590 catalog_manager.cc:1651] T 00000000000000000000000000000000 P fc4bbd8cf09e438895593b7651c63f06: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
16:20:15.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:15.925426 23194 heartbeater.cc:499] Master 127.22.10.61:38773 was elected leader, sending a full tablet report...
16:20:16.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:20:16.138528 23340 debug-util.cc:398] Leaking SignalData structure 0x7b08000b5e80 after lost signal to thread 23211
16:20:16.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:20:16.140578 23340 debug-util.cc:398] Leaking SignalData structure 0x7b08000ae9e0 after lost signal to thread 23343
16:20:16.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:15.038106 23559 server_base.cc:1061] running on GCE node
16:20:16.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:20:15.038511 23609 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:20:16.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:20:15.035720 23606 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:20:16.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:16.603333 23559 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:20:16.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:16.606768 23559 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:20:16.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:16.608196 23559 hybrid_clock.cc:648] HybridClock initialized: now 1790180416608166 us; error 45 us; skew 500 ppm
16:20:16.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:16.609130 23559 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:20:16.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 23 16:20:16 dist-test-slave-fqp8 krb5kdc[22580](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790180416, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.10.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:20:16.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:16.616835 23559 init.cc:377] Logged in from keytab as kudu/127.22.10.62@KRBTEST.COM (short username kudu)
16:20:16.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:16.640388 23559 webserver.cc:533] Webserver started at http://127.22.10.62:33841/ using document root <none> and password file <none>
16:20:16.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:16.641564 23559 fs_manager.cc:362] Metadata directory not provided
16:20:16.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:16.641896 23559 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:20:16.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:16.651298 23559 fs_manager.cc:714] Time spent opening directory manager: real 0.006s user 0.008s sys 0.000s
16:20:16.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:16.657187 23633 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:20:16.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:16.658458 23559 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.006s sys 0.000s
16:20:16.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:16.658828 23559 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/master-0/data,/tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/master-0/wal
16:20:16.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "083466e5de4a4ff682941e6b4624d3a0"
16:20:16.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-23 16:19:53 on dist-test-slave-fqp8"
16:20:16.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:16.661262 23559 fs_report.cc:389] FS layout report
16:20:16.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:20:16.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/master-0/wal
16:20:16.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/master-0/wal
16:20:16.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/master-0/data/data
16:20:16.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:20:16.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:20:16.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:20:16.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:20:16.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:20:16.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:20:16.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:20:16.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:20:16.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:20:16.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:20:16.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:20:16.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:16.698987 23344 heartbeater.cc:499] Master 127.22.10.61:38773 was elected leader, sending a full tablet report...
16:20:16.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:16.701916 23049 heartbeater.cc:499] Master 127.22.10.61:38773 was elected leader, sending a full tablet report...
16:20:16.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:16.776604 23559 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
16:20:16.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:16.782845 23559 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:20:16.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:16.783438 23559 kserver.cc:163] Server-wide thread pool size limit: 3276
16:20:16.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:20:16.901404 23389 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.22.10.62:37357: connect: Connection refused (error 111) [suppressed 7 similar messages]
16:20:16.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:16.991952 23559 rpc_server.cc:307] RPC server started. Bound to: 127.22.10.62:37357
16:20:16.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:16.992031 23688 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.22.10.62:37357 every 8 connection(s)
16:20:16.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:16.995389 23559 server_base.cc:1193] Dumped server information to /tmp/dist-test-tasko3a_Hu/test-tmp/mini-kudu-cluster6904808330888231834/master-0/data/info.pb
16:20:16.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:16.998272 22568 external_mini_cluster.cc:1434] Started /tmp/dist-test-tasko3a_Hu/build/tsan/bin/kudu as pid 23559
16:20:16.999 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
16:20:17.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:17.011018 23689 sys_catalog.cc:263] Verifying existing consensus state
16:20:17.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:17.018992 23689 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 083466e5de4a4ff682941e6b4624d3a0: Bootstrap starting.
16:20:17.040 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.22.10.61:38773(127.22.10.61:38773)] connecting to peer
16:20:17.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:17.074993 23689 log.cc:826] T 00000000000000000000000000000000 P 083466e5de4a4ff682941e6b4624d3a0: Log is configured to *not* fsync() on all Append() calls
16:20:17.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:17.097249 23689 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 083466e5de4a4ff682941e6b4624d3a0: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
16:20:17.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:17.098670 23689 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 083466e5de4a4ff682941e6b4624d3a0: Bootstrap complete.
16:20:17.113 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.22.10.60:39017(127.22.10.60:39017)] connecting to peer
16:20:17.118 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.22.10.62:37357(127.22.10.62:37357)] connecting to peer
16:20:17.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:17.135957 23689 raft_consensus.cc:359] T 00000000000000000000000000000000 P 083466e5de4a4ff682941e6b4624d3a0 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "083466e5de4a4ff682941e6b4624d3a0" member_type: VOTER last_known_addr { host: "127.22.10.62" port: 37357 } } peers { permanent_uuid: "9a7438b488774217a1a7eb7c632b81f8" member_type: VOTER last_known_addr { host: "127.22.10.61" port: 38773 } } peers { permanent_uuid: "fc4bbd8cf09e438895593b7651c63f06" member_type: VOTER last_known_addr { host: "127.22.10.60" port: 39017 } }
16:20:17.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:17.139693 23689 raft_consensus.cc:740] T 00000000000000000000000000000000 P 083466e5de4a4ff682941e6b4624d3a0 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 083466e5de4a4ff682941e6b4624d3a0, State: Initialized, Role: FOLLOWER
16:20:17.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:17.141695 23689 consensus_queue.cc:260] T 00000000000000000000000000000000 P 083466e5de4a4ff682941e6b4624d3a0 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "083466e5de4a4ff682941e6b4624d3a0" member_type: VOTER last_known_addr { host: "127.22.10.62" port: 37357 } } peers { permanent_uuid: "9a7438b488774217a1a7eb7c632b81f8" member_type: VOTER last_known_addr { host: "127.22.10.61" port: 38773 } } peers { permanent_uuid: "fc4bbd8cf09e438895593b7651c63f06" member_type: VOTER last_known_addr { host: "127.22.10.60" port: 39017 } }
16:20:17.150 [DEBUG - kudu-nio-2] (Connection.java:757) [peer master-127.22.10.62:37357(127.22.10.62:37357)] Successfully connected to peer
16:20:17.154 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.22.10.60:39017(127.22.10.60:39017)] Successfully connected to peer
16:20:17.159 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.22.10.61:38773(127.22.10.61:38773)] Successfully connected to peer
16:20:17.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:17.171518 23689 sys_catalog.cc:565] T 00000000000000000000000000000000 P 083466e5de4a4ff682941e6b4624d3a0 [sys.catalog]: configured and running, proceeding with master startup.
16:20:17.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 23 16:20:17 dist-test-slave-fqp8 krb5kdc[22580](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790180417, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.10.3@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:20:17.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:17.182390 23227 init.cc:295] Successfully reacquired a new kerberos TGT
16:20:17.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:17.182338 23700 sys_catalog.cc:455] T 00000000000000000000000000000000 P 083466e5de4a4ff682941e6b4624d3a0 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "083466e5de4a4ff682941e6b4624d3a0" member_type: VOTER last_known_addr { host: "127.22.10.62" port: 37357 } } peers { permanent_uuid: "9a7438b488774217a1a7eb7c632b81f8" member_type: VOTER last_known_addr { host: "127.22.10.61" port: 38773 } } peers { permanent_uuid: "fc4bbd8cf09e438895593b7651c63f06" member_type: VOTER last_known_addr { host: "127.22.10.60" port: 39017 } } }
16:20:17.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:17.183462 23700 sys_catalog.cc:458] T 00000000000000000000000000000000 P 083466e5de4a4ff682941e6b4624d3a0 [sys.catalog]: This master's current role is: FOLLOWER
16:20:17.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:17.285168 23712 catalog_manager.cc:1308] Loaded cluster ID: 2b04f3c8aee34f46a1410a54279e8a40
16:20:17.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:17.285435 23712 catalog_manager.cc:1601] T 00000000000000000000000000000000 P 083466e5de4a4ff682941e6b4624d3a0: loading cluster ID for follower catalog manager: success
16:20:17.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:17.294365 23712 catalog_manager.cc:1623] T 00000000000000000000000000000000 P 083466e5de4a4ff682941e6b4624d3a0: acquiring CA information for follower catalog manager: success
16:20:17.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 23 16:20:17 dist-test-slave-fqp8 krb5kdc[22580](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790180416, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.10.62@KRBTEST.COM for kudu/127.22.10.62@KRBTEST.COM
16:20:17.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:17.303966 23712 catalog_manager.cc:1651] T 00000000000000000000000000000000 P 083466e5de4a4ff682941e6b4624d3a0: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
16:20:17.315 [WARN - kudu-nio-2] (SecurityContext.java:206) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
16:20:17.318 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.22.10.62:37357(127.22.10.62:37357)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
16:20:17.319 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.10.61:38773(127.22.10.61:38773)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
16:20:17.319 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.22.10.62:37357: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
16:20:17.318 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.10.60:39017(127.22.10.60:39017)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
16:20:17.321 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.10.60:39017: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
16:20:17.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:20:17.323259 23699 negotiation.cc:336] Failed RPC negotiation. Trace:
16:20:17.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0923 16:20:17.142458 (+ 0us) reactor.cc:730] Submitting negotiation task for server connection from 127.0.0.1:57238 (local address 127.22.10.60:39017)
16:20:17.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0923 16:20:17.168202 (+ 25744us) server_negotiation.cc:207] Beginning negotiation
16:20:17.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0923 16:20:17.168230 (+ 28us) server_negotiation.cc:400] Waiting for connection header
16:20:17.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0923 16:20:17.246788 (+ 78558us) server_negotiation.cc:408] Connection header received
16:20:17.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0923 16:20:17.291254 (+ 44466us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
16:20:17.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0923 16:20:17.291284 (+ 30us) server_negotiation.cc:462] Received NEGOTIATE request from client
16:20:17.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0923 16:20:17.291505 (+ 221us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
16:20:17.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0923 16:20:17.291664 (+ 159us) server_negotiation.cc:227] Negotiated authn=SASL
16:20:17.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0923 16:20:17.321883 (+ 30219us) negotiation.cc:326] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:57238: BlockingRecv error: recv got EOF from 127.0.0.1:57238 (error 108)
16:20:17.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":25120,"thread_start_us":25731,"threads_started":1}
16:20:17.319 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.10.61:38773: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
16:20:17.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:20:17.327996 23698 negotiation.cc:336] Failed RPC negotiation. Trace:
16:20:17.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0923 16:20:17.139062 (+ 0us) reactor.cc:730] Submitting negotiation task for server connection from 127.0.0.1:58188 (local address 127.22.10.62:37357)
16:20:17.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0923 16:20:17.160065 (+ 21003us) server_negotiation.cc:207] Beginning negotiation
16:20:17.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0923 16:20:17.160114 (+ 49us) server_negotiation.cc:400] Waiting for connection header
16:20:17.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0923 16:20:17.246339 (+ 86225us) server_negotiation.cc:408] Connection header received
16:20:17.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0923 16:20:17.286401 (+ 40062us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
16:20:17.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0923 16:20:17.286434 (+ 33us) server_negotiation.cc:462] Received NEGOTIATE request from client
16:20:17.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0923 16:20:17.286658 (+ 224us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
16:20:17.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0923 16:20:17.286817 (+ 159us) server_negotiation.cc:227] Negotiated authn=SASL
16:20:17.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0923 16:20:17.320528 (+ 33711us) negotiation.cc:326] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:58188: BlockingRecv error: recv got EOF from 127.0.0.1:58188 (error 108)
16:20:17.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":19923,"thread_start_us":20890,"threads_started":1}
16:20:17.328 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.22.10.60:39017(127.22.10.60:39017)] handling channelInactive
16:20:17.329 [DEBUG - kudu-nio-2] (Connection.java:239) [peer master-127.22.10.62:37357(127.22.10.62:37357)] handling channelInactive
16:20:17.335 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1523) Notify RPC KuduRpc(method=ListTabletServers, tablet=null, attempt=1, TimeoutTracker(timeout=30000, elapsed=332), Traces: [0ms] refreshing cache from master, [24ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.10.61:38773, [107ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.10.60:39017, [112ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.10.62:37357, [313ms] Sub RPC ConnectToMaster: received response from server master-127.22.10.62:37357: Not authorized: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, [313ms] Sub RPC ConnectToMaster: received response from server master-127.22.10.61:38773: Not authorized: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, [315ms] Sub RPC ConnectToMaster: received response from server master-127.22.10.60:39017: Not authorized: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, deferred=null) after lookup exception
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.22.10.61:38773,127.22.10.60:39017,127.22.10.62:37357). Exceptions received: [org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:291)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:345)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
at java.base/java.lang.Thread.run(Thread.java:840)
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.22.10.61:38773,127.22.10.60:39017,127.22.10.62:37357). Exceptions received: [org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:291)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:345)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
at java.base/java.lang.Thread.run(Thread.java:840)
16:20:17.364 [DEBUG - kudu-nio-0] (KuduRpc.java:274) Handling a callback on RPC KuduRpc(method=ListTabletServers, tablet=null, attempt=1, TimeoutTracker(timeout=30000, elapsed=361), Traces: [0ms] refreshing cache from master, [24ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.10.61:38773, [107ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.10.60:39017, [112ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.10.62:37357, [313ms] Sub RPC ConnectToMaster: received response from server master-127.22.10.62:37357: Not authorized: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, [313ms] Sub RPC ConnectToMaster: received response from server master-127.22.10.61:38773: Not authorized: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, [315ms] Sub RPC ConnectToMaster: received response from server master-127.22.10.60:39017: Not authorized: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, deferred=null) with no deferred attached!
16:20:17.367 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.22.10.60:39017(127.22.10.60:39017)] disconnecting while in state TERMINATED
16:20:17.368 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.22.10.61:38773(127.22.10.61:38773)] disconnecting while in state TERMINATED
16:20:17.369 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.22.10.62:37357(127.22.10.62:37357)] disconnecting while in state TERMINATED
16:20:17.373 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2704) Releasing all remaining resources
16:20:17.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:20:17.375509 23697 negotiation.cc:336] Failed RPC negotiation. Trace:
16:20:17.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0923 16:20:17.137432 (+ 0us) reactor.cc:730] Submitting negotiation task for server connection from 127.0.0.1:43322 (local address 127.22.10.61:38773)
16:20:17.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0923 16:20:17.145902 (+ 8470us) server_negotiation.cc:207] Beginning negotiation
16:20:17.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0923 16:20:17.145938 (+ 36us) server_negotiation.cc:400] Waiting for connection header
16:20:17.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0923 16:20:17.246199 (+100261us) server_negotiation.cc:408] Connection header received
16:20:17.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0923 16:20:17.287914 (+ 41715us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
16:20:17.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0923 16:20:17.287945 (+ 31us) server_negotiation.cc:462] Received NEGOTIATE request from client
16:20:17.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0923 16:20:17.288181 (+ 236us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
16:20:17.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0923 16:20:17.288375 (+ 194us) server_negotiation.cc:227] Negotiated authn=SASL
16:20:17.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0923 16:20:17.374033 (+ 85658us) negotiation.cc:326] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:43322: BlockingRecv error: recv got EOF from 127.0.0.1:43322 (error 108)
16:20:17.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":7608,"thread_start_us":12736,"threads_started":1}
16:20:17.398 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.22.10.61:38773(127.22.10.61:38773)] handling channelInactive
16:20:17.400 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
16:20:17.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:17.413368 22568 external_mini_cluster.cc:1664] Killing /tmp/dist-test-tasko3a_Hu/build/tsan/bin/kudu with pid 22916
16:20:17.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:17.459170 22568 external_mini_cluster.cc:1664] Killing /tmp/dist-test-tasko3a_Hu/build/tsan/bin/kudu with pid 23063
16:20:17.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:17.495685 22568 external_mini_cluster.cc:1664] Killing /tmp/dist-test-tasko3a_Hu/build/tsan/bin/kudu with pid 23209
16:20:17.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:17.518517 23663 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 083466e5de4a4ff682941e6b4624d3a0 [term 1 FOLLOWER]: Advancing to term 2
16:20:17.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:17.534160 22568 external_mini_cluster.cc:1664] Killing /tmp/dist-test-tasko3a_Hu/build/tsan/bin/kudu with pid 23559
16:20:17.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:17.565030 22568 external_mini_cluster.cc:1664] Killing /tmp/dist-test-tasko3a_Hu/build/tsan/bin/kudu with pid 23359
16:20:17.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:17.602473 22568 external_mini_cluster.cc:1664] Killing /tmp/dist-test-tasko3a_Hu/build/tsan/bin/kudu with pid 23444
16:20:17.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:20:17.869585 22552 debug-util.cc:398] Leaking SignalData structure 0x7b08000b8c00 after lost signal to thread 22427
16:20:17.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 16:20:17.871038 22552 debug-util.cc:398] Leaking SignalData structure 0x7b08000b80a0 after lost signal to thread 22555
16:20:18.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-23T16:20:18Z chronyd exiting
16:20:19.857 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
16:20:19.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:19.860713 21831 external_mini_cluster.cc:1664] Killing /tmp/dist-test-tasko3a_Hu/build/tsan/bin/kudu with pid 22112
16:20:19.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:19.887295 21831 external_mini_cluster.cc:1664] Killing /tmp/dist-test-tasko3a_Hu/build/tsan/bin/kudu with pid 22278
16:20:19.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:19.911823 21831 external_mini_cluster.cc:1664] Killing /tmp/dist-test-tasko3a_Hu/build/tsan/bin/kudu with pid 22425
16:20:19.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:19.937067 21831 external_mini_cluster.cc:1664] Killing /tmp/dist-test-tasko3a_Hu/build/tsan/bin/kudu with pid 21843
16:20:19.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:19.969364 21831 external_mini_cluster.cc:1664] Killing /tmp/dist-test-tasko3a_Hu/build/tsan/bin/kudu with pid 21913
16:20:19.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 16:20:19.999078 21831 external_mini_cluster.cc:1664] Killing /tmp/dist-test-tasko3a_Hu/build/tsan/bin/kudu with pid 21988
16:20:21.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-23T16:20:21Z chronyd exiting
16:20:22.177 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.22.10.61:38773,127.22.10.60:39017,127.22.10.62:37357). Exceptions received: [org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available]
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:160)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:456)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:569)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at java.base/java.lang.Thread.run(Thread.java:840)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:291)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:345)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
... 1 more
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.22.10.61:38773,127.22.10.60:39017,127.22.10.62:37357). Exceptions received: [org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available]
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:160)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:456)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:569)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at java.base/java.lang.Thread.run(Thread.java:840)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:291)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:345)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
... 1 more