11:40:25.525 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
11:40:25.525 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-task4Z5jUl/test-tmp
11:40:25.525 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
11:40:25.525 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-task4Z5jUl/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
11:40:25.527 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster2916637498394586275"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
11:40:25.824 [DEBUG - main] (MiniKuduCluster.java:195) Response:
11:40:25.824 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
11:40:25.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:25.827080 18141 mini_oidc.cc:152] Starting JWKS server
11:40:25.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:25.830401 18141 webserver.cc:460] Webserver started at http://127.0.0.1:32991/ using document root <none> and password file <none>
11:40:25.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:25.830776 18141 mini_oidc.cc:189] Starting OIDC Discovery server
11:40:25.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:25.832329 18141 webserver.cc:460] Webserver started at http://127.0.0.1:40647/ using document root <none> and password file <none>
11:40:25.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-29T11:40:25Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
11:40:25.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-29T11:40:25Z Disabled control of system clock
11:40:25.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:25.875679 18141 external_mini_cluster.cc:1326] Running /tmp/dist-test-task4Z5jUl/build/tsan/bin/kudu
11:40:25.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task4Z5jUl/build/tsan/bin/kudu
11:40:25.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster2916637498394586275/master-0/wal
11:40:25.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster2916637498394586275/master-0/data
11:40:25.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
11:40:25.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
11:40:25.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
11:40:25.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:40:25.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:40:25.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:40:25.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster2916637498394586275/master-0/logs
11:40:25.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster2916637498394586275/master-0/data/info.pb
11:40:25.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:40:25.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
11:40:25.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
11:40:25.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
11:40:25.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
11:40:25.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:40:25.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:40:25.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:40:25.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
11:40:25.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
11:40:25.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
11:40:25.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
11:40:25.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.183.126:38399
11:40:25.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.183.126
11:40:25.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
11:40:25.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.183.84:45103
11:40:25.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:40:25.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:40:25.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:40:25.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
11:40:25.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.17.183.126:38399,127.17.183.125:40657,127.17.183.124:36315
11:40:25.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
11:40:25.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:32991/jwks.json with env {}
11:40:26.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:26.148871 18153 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
11:40:26.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:26.149374 18153 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
11:40:26.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:26.149829 18153 flags.cc:425] Enabled unsafe flag: --never_fsync=true
11:40:26.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:26.180037 18153 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
11:40:26.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:26.180289 18153 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
11:40:26.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:26.180506 18153 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
11:40:26.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:26.180694 18153 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
11:40:26.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:26.180873 18153 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
11:40:26.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:26.181006 18153 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:32991/jwks.json
11:40:26.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:26.214527 18153 master_runner.cc:386] Master server non-default flags:
11:40:26.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:40:26.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.183.84:45103
11:40:26.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:40:26.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:40:26.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster2916637498394586275/master-0/data
11:40:26.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster2916637498394586275/master-0/wal
11:40:26.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
11:40:26.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.17.183.126:38399,127.17.183.125:40657,127.17.183.124:36315
11:40:26.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:40:26.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:40:26.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
11:40:26.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.183.126:38399
11:40:26.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
11:40:26.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
11:40:26.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
11:40:26.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:32991/jwks.json
11:40:26.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:40:26.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:40:26.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster2916637498394586275/master-0/data/info.pb
11:40:26.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.183.126
11:40:26.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
11:40:26.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
11:40:26.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:40:26.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
11:40:26.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
11:40:26.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:40:26.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster2916637498394586275/master-0/logs
11:40:26.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:40:26.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
11:40:26.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
11:40:26.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
11:40:26.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
11:40:26.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
11:40:26.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
11:40:26.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 10:52:50 UTC on ccycloud.quasar-peqjre.root.comops.site
11:40:26.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679900
11:40:26.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
11:40:26.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:26.215682 18153 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
11:40:26.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:26.217140 18153 file_cache.cc:493] Constructed file cache file cache with capacity 419430
11:40:27.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:26.233626 18159 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:40:27.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:26.233645 18160 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:40:27.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:26.233654 18162 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:40:27.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:26.235045 18153 server_base.cc:1034] running on GCE node
11:40:27.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:27.315878 18153 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
11:40:27.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:27.318650 18153 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
11:40:27.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:27.320041 18153 hybrid_clock.cc:648] HybridClock initialized: now 1790682027320008 us; error 54 us; skew 500 ppm
11:40:27.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:27.320809 18153 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
11:40:27.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:27.332623 18153 webserver.cc:460] Webserver started at http://127.17.183.126:43951/ using document root <none> and password file <none>
11:40:27.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:27.333516 18153 fs_manager.cc:359] Metadata directory not provided
11:40:27.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:27.333693 18153 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
11:40:27.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:27.334062 18153 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
11:40:27.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:27.338591 18153 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster2916637498394586275/master-0/data/instance:
11:40:27.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d9b357dd6fea485e91203f1eef39aa33"
11:40:27.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 11:40:27 on dist-test-slave-h1fj"
11:40:27.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:27.339632 18153 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster2916637498394586275/master-0/wal/instance:
11:40:27.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d9b357dd6fea485e91203f1eef39aa33"
11:40:27.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 11:40:27 on dist-test-slave-h1fj"
11:40:27.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:27.346159 18153 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.005s sys 0.000s
11:40:27.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:27.351166 18169 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
11:40:27.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:27.352152 18153 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
11:40:27.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:27.352429 18153 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster2916637498394586275/master-0/data,/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster2916637498394586275/master-0/wal
11:40:27.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d9b357dd6fea485e91203f1eef39aa33"
11:40:27.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 11:40:27 on dist-test-slave-h1fj"
11:40:27.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:27.352699 18153 fs_report.cc:383] FS layout report
11:40:27.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
11:40:27.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster2916637498394586275/master-0/wal
11:40:27.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster2916637498394586275/master-0/wal
11:40:27.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster2916637498394586275/master-0/data/data
11:40:27.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
11:40:27.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
11:40:27.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
11:40:27.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
11:40:27.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
11:40:27.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
11:40:27.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
11:40:27.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
11:40:27.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
11:40:27.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
11:40:27.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
11:40:27.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:27.387583 18153 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
11:40:27.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:27.388921 18153 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
11:40:27.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:27.389295 18153 kserver.cc:163] Server-wide thread pool size limit: 3276
11:40:27.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:27.453945 18153 rpc_server.cc:307] RPC server started. Bound to: 127.17.183.126:38399
11:40:27.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:27.454043 18220 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.17.183.126:38399 every 8 connection(s)
11:40:27.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:27.456470 18153 server_base.cc:1166] Dumped server information to /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster2916637498394586275/master-0/data/info.pb
11:40:27.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:27.461212 18221 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
11:40:27.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:27.464233 18141 external_mini_cluster.cc:1388] Started /tmp/dist-test-task4Z5jUl/build/tsan/bin/kudu as pid 18153
11:40:27.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:27.464615 18141 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster2916637498394586275/master-0/wal/instance
11:40:27.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:27.467893 18141 external_mini_cluster.cc:1326] Running /tmp/dist-test-task4Z5jUl/build/tsan/bin/kudu
11:40:27.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task4Z5jUl/build/tsan/bin/kudu
11:40:27.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster2916637498394586275/master-1/wal
11:40:27.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster2916637498394586275/master-1/data
11:40:27.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
11:40:27.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
11:40:27.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
11:40:27.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:40:27.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:40:27.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:40:27.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster2916637498394586275/master-1/logs
11:40:27.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster2916637498394586275/master-1/data/info.pb
11:40:27.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:40:27.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
11:40:27.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
11:40:27.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
11:40:27.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
11:40:27.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:40:27.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:40:27.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:40:27.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
11:40:27.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
11:40:27.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
11:40:27.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
11:40:27.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.183.125:40657
11:40:27.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.183.125
11:40:27.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
11:40:27.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.183.84:45103
11:40:27.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:40:27.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:40:27.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:40:27.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
11:40:27.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.17.183.126:38399,127.17.183.125:40657,127.17.183.124:36315
11:40:27.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
11:40:27.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:32991/jwks.json with env {}
11:40:27.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:27.472918 18221 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.17.183.126" port: 38399 } has no permanent_uuid. Determining permanent_uuid...
11:40:27.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:27.501375 18221 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.17.183.125" port: 40657 } has no permanent_uuid. Determining permanent_uuid...
11:40:27.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:27.503710 18173 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.17.183.125:40657: connect: Connection refused (error 111)
11:40:27.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:27.507572 18221 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.17.183.125:40657: Network error: Client connection negotiation failed: client connection to 127.17.183.125:40657: connect: Connection refused (error 111)
11:40:27.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:27.557031 18221 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.183.125" port: 40657 } attempt: 1
11:40:27.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:27.560698 18221 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.17.183.125:40657: Network error: Client connection negotiation failed: client connection to 127.17.183.125:40657: connect: Connection refused (error 111)
11:40:27.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:27.629079 18221 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.183.125" port: 40657 } attempt: 2
11:40:27.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:27.632869 18221 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.17.183.125:40657: Network error: Client connection negotiation failed: client connection to 127.17.183.125:40657: connect: Connection refused (error 111)
11:40:27.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:27.724248 18221 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.183.125" port: 40657 } attempt: 3
11:40:27.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:27.727715 18221 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.17.183.125:40657: Network error: Client connection negotiation failed: client connection to 127.17.183.125:40657: connect: Connection refused (error 111)
11:40:27.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:27.754177 18223 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
11:40:27.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:27.754724 18223 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
11:40:27.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:27.755277 18223 flags.cc:425] Enabled unsafe flag: --never_fsync=true
11:40:27.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:27.785055 18223 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
11:40:27.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:27.785362 18223 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
11:40:27.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:27.785641 18223 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
11:40:27.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:27.785905 18223 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
11:40:27.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:27.786181 18223 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
11:40:27.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:27.786371 18223 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:32991/jwks.json
11:40:27.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:27.819080 18223 master_runner.cc:386] Master server non-default flags:
11:40:27.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:40:27.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.183.84:45103
11:40:27.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:40:27.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:40:27.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster2916637498394586275/master-1/data
11:40:27.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster2916637498394586275/master-1/wal
11:40:27.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
11:40:27.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.17.183.126:38399,127.17.183.125:40657,127.17.183.124:36315
11:40:27.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:40:27.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:40:27.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
11:40:27.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.183.125:40657
11:40:27.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
11:40:27.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
11:40:27.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
11:40:27.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:32991/jwks.json
11:40:27.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:40:27.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:40:27.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster2916637498394586275/master-1/data/info.pb
11:40:27.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.183.125
11:40:27.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
11:40:27.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
11:40:27.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:40:27.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
11:40:27.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
11:40:27.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:40:27.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster2916637498394586275/master-1/logs
11:40:27.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:40:27.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
11:40:27.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
11:40:27.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
11:40:27.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
11:40:27.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
11:40:27.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
11:40:27.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 10:52:50 UTC on ccycloud.quasar-peqjre.root.comops.site
11:40:27.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679900
11:40:27.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
11:40:27.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:27.820468 18223 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
11:40:27.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:27.822067 18223 file_cache.cc:493] Constructed file cache file cache with capacity 419430
11:40:27.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:27.837947 18233 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:40:27.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:27.871058 18221 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.183.125" port: 40657 } attempt: 4
11:40:27.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:27.874826 18221 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.17.183.125:40657: Network error: Client connection negotiation failed: client connection to 127.17.183.125:40657: connect: Connection refused (error 111)
11:40:28.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:28.174166 18221 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.183.125" port: 40657 } attempt: 5
11:40:28.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:28.178002 18221 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.17.183.125:40657: Network error: Client connection negotiation failed: client connection to 127.17.183.125:40657: connect: Connection refused (error 111)
11:40:28.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:28.725350 18221 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.183.125" port: 40657 } attempt: 6
11:40:28.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:28.734707 18221 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.17.183.125:40657: Network error: Client connection negotiation failed: client connection to 127.17.183.125:40657: connect: Connection refused (error 111)
11:40:28.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:27.837985 18235 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:40:28.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:27.838013 18232 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:40:28.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:27.839509 18223 server_base.cc:1034] running on GCE node
11:40:28.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:28.922348 18223 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
11:40:28.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:28.925179 18223 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
11:40:28.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:28.926590 18223 hybrid_clock.cc:648] HybridClock initialized: now 1790682028926536 us; error 72 us; skew 500 ppm
11:40:28.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:28.927423 18223 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
11:40:28.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:28.933586 18223 webserver.cc:460] Webserver started at http://127.17.183.125:45007/ using document root <none> and password file <none>
11:40:28.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:28.934501 18223 fs_manager.cc:359] Metadata directory not provided
11:40:28.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:28.934679 18223 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
11:40:28.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:28.935055 18223 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
11:40:28.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:28.939778 18223 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster2916637498394586275/master-1/data/instance:
11:40:28.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "60515cb33c7343f780390d2a26bc1b94"
11:40:28.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 11:40:28 on dist-test-slave-h1fj"
11:40:28.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:28.940788 18223 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster2916637498394586275/master-1/wal/instance:
11:40:28.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "60515cb33c7343f780390d2a26bc1b94"
11:40:28.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 11:40:28 on dist-test-slave-h1fj"
11:40:28.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:28.947372 18223 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.007s sys 0.001s
11:40:28.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:28.952517 18244 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
11:40:28.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:28.953480 18223 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.005s sys 0.000s
11:40:28.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:28.953754 18223 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster2916637498394586275/master-1/data,/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster2916637498394586275/master-1/wal
11:40:28.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "60515cb33c7343f780390d2a26bc1b94"
11:40:28.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 11:40:28 on dist-test-slave-h1fj"
11:40:28.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:28.954068 18223 fs_report.cc:383] FS layout report
11:40:28.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
11:40:28.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster2916637498394586275/master-1/wal
11:40:28.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster2916637498394586275/master-1/wal
11:40:28.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster2916637498394586275/master-1/data/data
11:40:28.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
11:40:28.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
11:40:28.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
11:40:28.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
11:40:28.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
11:40:28.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
11:40:28.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
11:40:28.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
11:40:28.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
11:40:28.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
11:40:28.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
11:40:28.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:28.998544 18223 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
11:40:29.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:28.999980 18223 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
11:40:29.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:29.000365 18223 kserver.cc:163] Server-wide thread pool size limit: 3276
11:40:29.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:29.064091 18223 rpc_server.cc:307] RPC server started. Bound to: 127.17.183.125:40657
11:40:29.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:29.064193 18295 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.17.183.125:40657 every 8 connection(s)
11:40:29.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:29.066577 18223 server_base.cc:1166] Dumped server information to /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster2916637498394586275/master-1/data/info.pb
11:40:29.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:29.070003 18141 external_mini_cluster.cc:1388] Started /tmp/dist-test-task4Z5jUl/build/tsan/bin/kudu as pid 18223
11:40:29.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:29.070464 18141 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster2916637498394586275/master-1/wal/instance
11:40:29.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:29.072589 18141 external_mini_cluster.cc:1326] Running /tmp/dist-test-task4Z5jUl/build/tsan/bin/kudu
11:40:29.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task4Z5jUl/build/tsan/bin/kudu
11:40:29.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster2916637498394586275/master-2/wal
11:40:29.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster2916637498394586275/master-2/data
11:40:29.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
11:40:29.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
11:40:29.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
11:40:29.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:40:29.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:40:29.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:40:29.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster2916637498394586275/master-2/logs
11:40:29.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster2916637498394586275/master-2/data/info.pb
11:40:29.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:40:29.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
11:40:29.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
11:40:29.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
11:40:29.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
11:40:29.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:40:29.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:40:29.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:40:29.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
11:40:29.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
11:40:29.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
11:40:29.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
11:40:29.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.183.124:36315
11:40:29.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.183.124
11:40:29.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
11:40:29.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.183.84:45103
11:40:29.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:40:29.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:40:29.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:40:29.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
11:40:29.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.17.183.126:38399,127.17.183.125:40657,127.17.183.124:36315
11:40:29.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
11:40:29.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:32991/jwks.json with env {}
11:40:29.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:29.072849 18296 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
11:40:29.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:29.083200 18296 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.17.183.126" port: 38399 } has no permanent_uuid. Determining permanent_uuid...
11:40:29.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:29.107162 18296 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.17.183.125" port: 40657 } has no permanent_uuid. Determining permanent_uuid...
11:40:29.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:29.122670 18296 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.17.183.124" port: 36315 } has no permanent_uuid. Determining permanent_uuid...
11:40:29.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:29.124701 18245 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.17.183.124:36315: connect: Connection refused (error 111)
11:40:29.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:29.128300 18296 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.17.183.124:36315: Network error: Client connection negotiation failed: client connection to 127.17.183.124:36315: connect: Connection refused (error 111)
11:40:29.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:29.177693 18296 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.183.124" port: 36315 } attempt: 1
11:40:29.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:29.180925 18296 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.17.183.124:36315: Network error: Client connection negotiation failed: client connection to 127.17.183.124:36315: connect: Connection refused (error 111)
11:40:29.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:29.249231 18296 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.183.124" port: 36315 } attempt: 2
11:40:29.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:29.252406 18296 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.17.183.124:36315: Network error: Client connection negotiation failed: client connection to 127.17.183.124:36315: connect: Connection refused (error 111)
11:40:29.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:29.343740 18296 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.183.124" port: 36315 } attempt: 3
11:40:29.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:29.346936 18296 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.17.183.124:36315: Network error: Client connection negotiation failed: client connection to 127.17.183.124:36315: connect: Connection refused (error 111)
11:40:29.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:29.361564 18298 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
11:40:29.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:29.362103 18298 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
11:40:29.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:29.362659 18298 flags.cc:425] Enabled unsafe flag: --never_fsync=true
11:40:29.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:29.391925 18298 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
11:40:29.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:29.392235 18298 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
11:40:29.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:29.392500 18298 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
11:40:29.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:29.392755 18298 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
11:40:29.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:29.393019 18298 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
11:40:29.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:29.393242 18298 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:32991/jwks.json
11:40:29.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:29.425896 18298 master_runner.cc:386] Master server non-default flags:
11:40:29.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:40:29.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.183.84:45103
11:40:29.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:40:29.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:40:29.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster2916637498394586275/master-2/data
11:40:29.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster2916637498394586275/master-2/wal
11:40:29.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
11:40:29.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.17.183.126:38399,127.17.183.125:40657,127.17.183.124:36315
11:40:29.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:40:29.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:40:29.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
11:40:29.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.183.124:36315
11:40:29.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
11:40:29.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
11:40:29.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
11:40:29.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:32991/jwks.json
11:40:29.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:40:29.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:40:29.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster2916637498394586275/master-2/data/info.pb
11:40:29.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.183.124
11:40:29.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
11:40:29.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
11:40:29.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:40:29.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
11:40:29.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
11:40:29.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:40:29.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster2916637498394586275/master-2/logs
11:40:29.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:40:29.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
11:40:29.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
11:40:29.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
11:40:29.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
11:40:29.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
11:40:29.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
11:40:29.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 10:52:50 UTC on ccycloud.quasar-peqjre.root.comops.site
11:40:29.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679900
11:40:29.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
11:40:29.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:29.427265 18298 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
11:40:29.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:29.428907 18298 file_cache.cc:493] Constructed file cache file cache with capacity 419430
11:40:29.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:29.445287 18309 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:40:29.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:29.490284 18296 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.183.124" port: 36315 } attempt: 4
11:40:29.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:29.493535 18296 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.17.183.124:36315: Network error: Client connection negotiation failed: client connection to 127.17.183.124:36315: connect: Connection refused (error 111)
11:40:29.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:29.793026 18296 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.183.124" port: 36315 } attempt: 5
11:40:29.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:29.795114 18221 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.183.125" port: 40657 } attempt: 7
11:40:29.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:29.797422 18296 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.17.183.124:36315: Network error: Client connection negotiation failed: client connection to 127.17.183.124:36315: connect: Connection refused (error 111)
11:40:29.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:29.808089 18221 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.17.183.124" port: 36315 } has no permanent_uuid. Determining permanent_uuid...
11:40:29.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:29.816727 18221 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.17.183.124:36315: Network error: Client connection negotiation failed: client connection to 127.17.183.124:36315: connect: Connection refused (error 111)
11:40:29.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:29.875165 18221 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.183.124" port: 36315 } attempt: 1
11:40:29.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:29.878465 18221 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.17.183.124:36315: Network error: Client connection negotiation failed: client connection to 127.17.183.124:36315: connect: Connection refused (error 111)
11:40:29.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:29.959916 18221 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.183.124" port: 36315 } attempt: 2
11:40:29.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:29.963516 18221 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.17.183.124:36315: Network error: Client connection negotiation failed: client connection to 127.17.183.124:36315: connect: Connection refused (error 111)
11:40:30.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:30.048969 18221 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.183.124" port: 36315 } attempt: 3
11:40:30.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:30.052523 18221 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.17.183.124:36315: Network error: Client connection negotiation failed: client connection to 127.17.183.124:36315: connect: Connection refused (error 111)
11:40:30.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:30.192984 18221 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.183.124" port: 36315 } attempt: 4
11:40:30.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:30.196583 18221 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.17.183.124:36315: Network error: Client connection negotiation failed: client connection to 127.17.183.124:36315: connect: Connection refused (error 111)
11:40:30.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:30.344899 18296 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.183.124" port: 36315 } attempt: 6
11:40:30.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:30.350773 18296 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.17.183.124:36315: Network error: Client connection negotiation failed: client connection to 127.17.183.124:36315: connect: Connection refused (error 111)
11:40:30.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:30.480026 18221 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.183.124" port: 36315 } attempt: 5
11:40:30.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:30.483299 18221 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.17.183.124:36315: Network error: Client connection negotiation failed: client connection to 127.17.183.124:36315: connect: Connection refused (error 111)
11:40:30.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:29.445298 18308 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:40:30.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:29.447441 18298 server_base.cc:1034] running on GCE node
11:40:30.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:29.445524 18311 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:40:30.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:30.540141 18298 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
11:40:30.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:30.542574 18298 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
11:40:30.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:30.543918 18298 hybrid_clock.cc:648] HybridClock initialized: now 1790682030543871 us; error 39 us; skew 500 ppm
11:40:30.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:30.544724 18298 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
11:40:30.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:30.550266 18298 webserver.cc:460] Webserver started at http://127.17.183.124:38717/ using document root <none> and password file <none>
11:40:30.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:30.551132 18298 fs_manager.cc:359] Metadata directory not provided
11:40:30.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:30.551362 18298 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
11:40:30.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:30.551849 18298 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
11:40:30.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:30.556429 18298 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster2916637498394586275/master-2/data/instance:
11:40:30.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f2e1a3ae51974159a8647773b8f68deb"
11:40:30.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 11:40:30 on dist-test-slave-h1fj"
11:40:30.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:30.557480 18298 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster2916637498394586275/master-2/wal/instance:
11:40:30.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f2e1a3ae51974159a8647773b8f68deb"
11:40:30.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 11:40:30 on dist-test-slave-h1fj"
11:40:30.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:30.564126 18298 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.008s sys 0.000s
11:40:30.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:30.569337 18323 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
11:40:30.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:30.570518 18298 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.004s sys 0.001s
11:40:30.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:30.570797 18298 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster2916637498394586275/master-2/data,/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster2916637498394586275/master-2/wal
11:40:30.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f2e1a3ae51974159a8647773b8f68deb"
11:40:30.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 11:40:30 on dist-test-slave-h1fj"
11:40:30.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:30.571095 18298 fs_report.cc:383] FS layout report
11:40:30.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
11:40:30.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster2916637498394586275/master-2/wal
11:40:30.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster2916637498394586275/master-2/wal
11:40:30.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster2916637498394586275/master-2/data/data
11:40:30.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
11:40:30.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
11:40:30.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
11:40:30.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
11:40:30.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
11:40:30.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
11:40:30.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
11:40:30.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
11:40:30.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
11:40:30.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
11:40:30.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
11:40:30.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:30.617488 18298 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
11:40:30.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:30.618788 18298 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
11:40:30.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:30.619164 18298 kserver.cc:163] Server-wide thread pool size limit: 3276
11:40:30.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:30.683051 18298 rpc_server.cc:307] RPC server started. Bound to: 127.17.183.124:36315
11:40:30.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:30.683122 18374 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.17.183.124:36315 every 8 connection(s)
11:40:30.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:30.685560 18298 server_base.cc:1166] Dumped server information to /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster2916637498394586275/master-2/data/info.pb
11:40:30.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:30.690440 18375 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
11:40:30.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:30.692256 18141 external_mini_cluster.cc:1388] Started /tmp/dist-test-task4Z5jUl/build/tsan/bin/kudu as pid 18298
11:40:30.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:30.692572 18141 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster2916637498394586275/master-2/wal/instance
11:40:30.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:30.703925 18375 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.17.183.126" port: 38399 } has no permanent_uuid. Determining permanent_uuid...
11:40:30.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:30.733574 18375 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.17.183.125" port: 40657 } has no permanent_uuid. Determining permanent_uuid...
11:40:30.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:30.743040 18375 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.17.183.124" port: 36315 } has no permanent_uuid. Determining permanent_uuid...
11:40:30.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:30.758525 18375 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f2e1a3ae51974159a8647773b8f68deb: Bootstrap starting.
11:40:30.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:30.764392 18375 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P f2e1a3ae51974159a8647773b8f68deb: Neither blocks nor log segments found. Creating new log.
11:40:30.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:30.766100 18375 log.cc:824] T 00000000000000000000000000000000 P f2e1a3ae51974159a8647773b8f68deb: Log is configured to *not* fsync() on all Append() calls
11:40:30.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:30.770772 18375 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f2e1a3ae51974159a8647773b8f68deb: No bootstrap required, opened a new log
11:40:30.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:30.790585 18375 raft_consensus.cc:348] T 00000000000000000000000000000000 P f2e1a3ae51974159a8647773b8f68deb [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d9b357dd6fea485e91203f1eef39aa33" member_type: VOTER last_known_addr { host: "127.17.183.126" port: 38399 } } peers { permanent_uuid: "60515cb33c7343f780390d2a26bc1b94" member_type: VOTER last_known_addr { host: "127.17.183.125" port: 40657 } } peers { permanent_uuid: "f2e1a3ae51974159a8647773b8f68deb" member_type: VOTER last_known_addr { host: "127.17.183.124" port: 36315 } }
11:40:30.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:30.791129 18375 raft_consensus.cc:374] T 00000000000000000000000000000000 P f2e1a3ae51974159a8647773b8f68deb [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
11:40:30.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:30.791342 18375 raft_consensus.cc:729] T 00000000000000000000000000000000 P f2e1a3ae51974159a8647773b8f68deb [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f2e1a3ae51974159a8647773b8f68deb, State: Initialized, Role: FOLLOWER
11:40:30.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:30.791934 18375 consensus_queue.cc:260] T 00000000000000000000000000000000 P f2e1a3ae51974159a8647773b8f68deb [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d9b357dd6fea485e91203f1eef39aa33" member_type: VOTER last_known_addr { host: "127.17.183.126" port: 38399 } } peers { permanent_uuid: "60515cb33c7343f780390d2a26bc1b94" member_type: VOTER last_known_addr { host: "127.17.183.125" port: 40657 } } peers { permanent_uuid: "f2e1a3ae51974159a8647773b8f68deb" member_type: VOTER last_known_addr { host: "127.17.183.124" port: 36315 } }
11:40:30.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:30.794075 18384 sys_catalog.cc:455] T 00000000000000000000000000000000 P f2e1a3ae51974159a8647773b8f68deb [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d9b357dd6fea485e91203f1eef39aa33" member_type: VOTER last_known_addr { host: "127.17.183.126" port: 38399 } } peers { permanent_uuid: "60515cb33c7343f780390d2a26bc1b94" member_type: VOTER last_known_addr { host: "127.17.183.125" port: 40657 } } peers { permanent_uuid: "f2e1a3ae51974159a8647773b8f68deb" member_type: VOTER last_known_addr { host: "127.17.183.124" port: 36315 } } }
11:40:30.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:30.794682 18384 sys_catalog.cc:458] T 00000000000000000000000000000000 P f2e1a3ae51974159a8647773b8f68deb [sys.catalog]: This master's current role is: FOLLOWER
11:40:30.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:30.795228 18375 sys_catalog.cc:564] T 00000000000000000000000000000000 P f2e1a3ae51974159a8647773b8f68deb [sys.catalog]: configured and running, proceeding with master startup.
11:40:30.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:30.816936 18395 catalog_manager.cc:1558] T 00000000000000000000000000000000 P f2e1a3ae51974159a8647773b8f68deb: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:40:30.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:30.817171 18395 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:40:30.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:30.814101 18298 master_runner.cc:186] Error getting master registration for 127.17.183.126:38399: OK, instance_id { permanent_uuid: "d9b357dd6fea485e91203f1eef39aa33" instance_seqno: 1790682027436584 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
11:40:30.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:30.839708 18298 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
11:40:30.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:30.873808 18329 tablet.cc:2369] T 00000000000000000000000000000000 P f2e1a3ae51974159a8647773b8f68deb: Can't schedule compaction. Clean time has not been advanced past its initial value.
11:40:31.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:31.035712 18221 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.183.124" port: 36315 } attempt: 6
11:40:31.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:31.056237 18221 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d9b357dd6fea485e91203f1eef39aa33: Bootstrap starting.
11:40:31.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:31.063071 18221 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P d9b357dd6fea485e91203f1eef39aa33: Neither blocks nor log segments found. Creating new log.
11:40:31.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:31.065560 18221 log.cc:824] T 00000000000000000000000000000000 P d9b357dd6fea485e91203f1eef39aa33: Log is configured to *not* fsync() on all Append() calls
11:40:31.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:31.071328 18221 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d9b357dd6fea485e91203f1eef39aa33: No bootstrap required, opened a new log
11:40:31.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:31.100747 18221 raft_consensus.cc:348] T 00000000000000000000000000000000 P d9b357dd6fea485e91203f1eef39aa33 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d9b357dd6fea485e91203f1eef39aa33" member_type: VOTER last_known_addr { host: "127.17.183.126" port: 38399 } } peers { permanent_uuid: "60515cb33c7343f780390d2a26bc1b94" member_type: VOTER last_known_addr { host: "127.17.183.125" port: 40657 } } peers { permanent_uuid: "f2e1a3ae51974159a8647773b8f68deb" member_type: VOTER last_known_addr { host: "127.17.183.124" port: 36315 } }
11:40:31.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:31.101560 18221 raft_consensus.cc:374] T 00000000000000000000000000000000 P d9b357dd6fea485e91203f1eef39aa33 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
11:40:31.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:31.101850 18221 raft_consensus.cc:729] T 00000000000000000000000000000000 P d9b357dd6fea485e91203f1eef39aa33 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d9b357dd6fea485e91203f1eef39aa33, State: Initialized, Role: FOLLOWER
11:40:31.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:31.102715 18221 consensus_queue.cc:260] T 00000000000000000000000000000000 P d9b357dd6fea485e91203f1eef39aa33 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d9b357dd6fea485e91203f1eef39aa33" member_type: VOTER last_known_addr { host: "127.17.183.126" port: 38399 } } peers { permanent_uuid: "60515cb33c7343f780390d2a26bc1b94" member_type: VOTER last_known_addr { host: "127.17.183.125" port: 40657 } } peers { permanent_uuid: "f2e1a3ae51974159a8647773b8f68deb" member_type: VOTER last_known_addr { host: "127.17.183.124" port: 36315 } }
11:40:31.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:31.105353 18399 sys_catalog.cc:455] T 00000000000000000000000000000000 P d9b357dd6fea485e91203f1eef39aa33 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d9b357dd6fea485e91203f1eef39aa33" member_type: VOTER last_known_addr { host: "127.17.183.126" port: 38399 } } peers { permanent_uuid: "60515cb33c7343f780390d2a26bc1b94" member_type: VOTER last_known_addr { host: "127.17.183.125" port: 40657 } } peers { permanent_uuid: "f2e1a3ae51974159a8647773b8f68deb" member_type: VOTER last_known_addr { host: "127.17.183.124" port: 36315 } } }
11:40:31.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:31.105917 18399 sys_catalog.cc:458] T 00000000000000000000000000000000 P d9b357dd6fea485e91203f1eef39aa33 [sys.catalog]: This master's current role is: FOLLOWER
11:40:31.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:31.107304 18221 sys_catalog.cc:564] T 00000000000000000000000000000000 P d9b357dd6fea485e91203f1eef39aa33 [sys.catalog]: configured and running, proceeding with master startup.
11:40:31.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:31.131603 18410 catalog_manager.cc:1558] T 00000000000000000000000000000000 P d9b357dd6fea485e91203f1eef39aa33: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:40:31.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:31.132047 18410 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:40:31.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:31.146934 18175 tablet.cc:2369] T 00000000000000000000000000000000 P d9b357dd6fea485e91203f1eef39aa33: Can't schedule compaction. Clean time has not been advanced past its initial value.
11:40:31.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:31.131399 18153 master_runner.cc:186] Error getting master registration for 127.17.183.125:40657: OK, instance_id { permanent_uuid: "60515cb33c7343f780390d2a26bc1b94" instance_seqno: 1790682029046656 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
11:40:31.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:31.162184 18153 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
11:40:31.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:31.185093 18384 raft_consensus.cc:482] T 00000000000000000000000000000000 P f2e1a3ae51974159a8647773b8f68deb [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
11:40:31.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:31.185626 18384 raft_consensus.cc:504] T 00000000000000000000000000000000 P f2e1a3ae51974159a8647773b8f68deb [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d9b357dd6fea485e91203f1eef39aa33" member_type: VOTER last_known_addr { host: "127.17.183.126" port: 38399 } } peers { permanent_uuid: "60515cb33c7343f780390d2a26bc1b94" member_type: VOTER last_known_addr { host: "127.17.183.125" port: 40657 } } peers { permanent_uuid: "f2e1a3ae51974159a8647773b8f68deb" member_type: VOTER last_known_addr { host: "127.17.183.124" port: 36315 } }
11:40:31.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:31.188580 18384 leader_election.cc:288] T 00000000000000000000000000000000 P f2e1a3ae51974159a8647773b8f68deb [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers d9b357dd6fea485e91203f1eef39aa33 (127.17.183.126:38399), 60515cb33c7343f780390d2a26bc1b94 (127.17.183.125:40657)
11:40:31.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:31.189049 18196 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "f2e1a3ae51974159a8647773b8f68deb" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d9b357dd6fea485e91203f1eef39aa33" is_pre_election: true
11:40:31.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:31.189868 18196 raft_consensus.cc:2454] T 00000000000000000000000000000000 P d9b357dd6fea485e91203f1eef39aa33 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate f2e1a3ae51974159a8647773b8f68deb in term 0.
11:40:31.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:31.191191 18326 leader_election.cc:302] T 00000000000000000000000000000000 P f2e1a3ae51974159a8647773b8f68deb [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: d9b357dd6fea485e91203f1eef39aa33, f2e1a3ae51974159a8647773b8f68deb; no voters:
11:40:31.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:31.192004 18384 raft_consensus.cc:2781] T 00000000000000000000000000000000 P f2e1a3ae51974159a8647773b8f68deb [term 0 FOLLOWER]: Leader pre-election won for term 1
11:40:31.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:31.192317 18384 raft_consensus.cc:482] T 00000000000000000000000000000000 P f2e1a3ae51974159a8647773b8f68deb [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
11:40:31.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:31.192618 18384 raft_consensus.cc:3037] T 00000000000000000000000000000000 P f2e1a3ae51974159a8647773b8f68deb [term 0 FOLLOWER]: Advancing to term 1
11:40:31.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:31.198786 18384 raft_consensus.cc:504] T 00000000000000000000000000000000 P f2e1a3ae51974159a8647773b8f68deb [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d9b357dd6fea485e91203f1eef39aa33" member_type: VOTER last_known_addr { host: "127.17.183.126" port: 38399 } } peers { permanent_uuid: "60515cb33c7343f780390d2a26bc1b94" member_type: VOTER last_known_addr { host: "127.17.183.125" port: 40657 } } peers { permanent_uuid: "f2e1a3ae51974159a8647773b8f68deb" member_type: VOTER last_known_addr { host: "127.17.183.124" port: 36315 } }
11:40:31.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:31.200531 18384 leader_election.cc:288] T 00000000000000000000000000000000 P f2e1a3ae51974159a8647773b8f68deb [CANDIDATE]: Term 1 election: Requested vote from peers d9b357dd6fea485e91203f1eef39aa33 (127.17.183.126:38399), 60515cb33c7343f780390d2a26bc1b94 (127.17.183.125:40657)
11:40:31.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:31.201161 18196 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "f2e1a3ae51974159a8647773b8f68deb" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d9b357dd6fea485e91203f1eef39aa33"
11:40:31.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:31.201649 18196 raft_consensus.cc:3037] T 00000000000000000000000000000000 P d9b357dd6fea485e91203f1eef39aa33 [term 0 FOLLOWER]: Advancing to term 1
11:40:31.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:31.208006 18196 raft_consensus.cc:2454] T 00000000000000000000000000000000 P d9b357dd6fea485e91203f1eef39aa33 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate f2e1a3ae51974159a8647773b8f68deb in term 1.
11:40:31.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:31.210397 18326 leader_election.cc:302] T 00000000000000000000000000000000 P f2e1a3ae51974159a8647773b8f68deb [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: d9b357dd6fea485e91203f1eef39aa33, f2e1a3ae51974159a8647773b8f68deb; no voters:
11:40:31.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:31.216110 18384 raft_consensus.cc:2781] T 00000000000000000000000000000000 P f2e1a3ae51974159a8647773b8f68deb [term 1 FOLLOWER]: Leader election won for term 1
11:40:31.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:31.221032 18384 raft_consensus.cc:686] T 00000000000000000000000000000000 P f2e1a3ae51974159a8647773b8f68deb [term 1 LEADER]: Becoming Leader. State: Replica: f2e1a3ae51974159a8647773b8f68deb, State: Running, Role: LEADER
11:40:31.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:31.222007 18384 consensus_queue.cc:237] T 00000000000000000000000000000000 P f2e1a3ae51974159a8647773b8f68deb [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d9b357dd6fea485e91203f1eef39aa33" member_type: VOTER last_known_addr { host: "127.17.183.126" port: 38399 } } peers { permanent_uuid: "60515cb33c7343f780390d2a26bc1b94" member_type: VOTER last_known_addr { host: "127.17.183.125" port: 40657 } } peers { permanent_uuid: "f2e1a3ae51974159a8647773b8f68deb" member_type: VOTER last_known_addr { host: "127.17.183.124" port: 36315 } }
11:40:31.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:31.233076 18412 sys_catalog.cc:455] T 00000000000000000000000000000000 P f2e1a3ae51974159a8647773b8f68deb [sys.catalog]: SysCatalogTable state changed. Reason: New leader f2e1a3ae51974159a8647773b8f68deb. Latest consensus state: current_term: 1 leader_uuid: "f2e1a3ae51974159a8647773b8f68deb" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d9b357dd6fea485e91203f1eef39aa33" member_type: VOTER last_known_addr { host: "127.17.183.126" port: 38399 } } peers { permanent_uuid: "60515cb33c7343f780390d2a26bc1b94" member_type: VOTER last_known_addr { host: "127.17.183.125" port: 40657 } } peers { permanent_uuid: "f2e1a3ae51974159a8647773b8f68deb" member_type: VOTER last_known_addr { host: "127.17.183.124" port: 36315 } } }
11:40:31.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:31.233848 18412 sys_catalog.cc:458] T 00000000000000000000000000000000 P f2e1a3ae51974159a8647773b8f68deb [sys.catalog]: This master's current role is: LEADER
11:40:31.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:31.236819 18414 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
11:40:31.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:31.243801 18414 catalog_manager.cc:1484] Initializing Kudu cluster ID...
11:40:31.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:31.271184 18196 raft_consensus.cc:1261] T 00000000000000000000000000000000 P d9b357dd6fea485e91203f1eef39aa33 [term 1 FOLLOWER]: Refusing update from remote peer f2e1a3ae51974159a8647773b8f68deb: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
11:40:31.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:31.273029 18412 consensus_queue.cc:1035] T 00000000000000000000000000000000 P f2e1a3ae51974159a8647773b8f68deb [LEADER]: Connected to new peer: Peer: permanent_uuid: "d9b357dd6fea485e91203f1eef39aa33" member_type: VOTER last_known_addr { host: "127.17.183.126" port: 38399 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
11:40:31.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:31.297843 18399 sys_catalog.cc:455] T 00000000000000000000000000000000 P d9b357dd6fea485e91203f1eef39aa33 [sys.catalog]: SysCatalogTable state changed. Reason: New leader f2e1a3ae51974159a8647773b8f68deb. Latest consensus state: current_term: 1 leader_uuid: "f2e1a3ae51974159a8647773b8f68deb" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d9b357dd6fea485e91203f1eef39aa33" member_type: VOTER last_known_addr { host: "127.17.183.126" port: 38399 } } peers { permanent_uuid: "60515cb33c7343f780390d2a26bc1b94" member_type: VOTER last_known_addr { host: "127.17.183.125" port: 40657 } } peers { permanent_uuid: "f2e1a3ae51974159a8647773b8f68deb" member_type: VOTER last_known_addr { host: "127.17.183.124" port: 36315 } } }
11:40:31.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:31.298540 18399 sys_catalog.cc:458] T 00000000000000000000000000000000 P d9b357dd6fea485e91203f1eef39aa33 [sys.catalog]: This master's current role is: FOLLOWER
11:40:31.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:31.305027 18415 mvcc.cc:205] Tried to move back new op lower bound from 7334633600061874176 to 7334633599895031808. Current Snapshot: MvccSnapshot[applied={T|T < 7334633600061874176}]
11:40:31.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:31.308176 18384 sys_catalog.cc:455] T 00000000000000000000000000000000 P f2e1a3ae51974159a8647773b8f68deb [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "f2e1a3ae51974159a8647773b8f68deb" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d9b357dd6fea485e91203f1eef39aa33" member_type: VOTER last_known_addr { host: "127.17.183.126" port: 38399 } } peers { permanent_uuid: "60515cb33c7343f780390d2a26bc1b94" member_type: VOTER last_known_addr { host: "127.17.183.125" port: 40657 } } peers { permanent_uuid: "f2e1a3ae51974159a8647773b8f68deb" member_type: VOTER last_known_addr { host: "127.17.183.124" port: 36315 } } }
11:40:31.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:31.308914 18384 sys_catalog.cc:458] T 00000000000000000000000000000000 P f2e1a3ae51974159a8647773b8f68deb [sys.catalog]: This master's current role is: LEADER
11:40:31.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:31.310883 18399 sys_catalog.cc:455] T 00000000000000000000000000000000 P d9b357dd6fea485e91203f1eef39aa33 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "f2e1a3ae51974159a8647773b8f68deb" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d9b357dd6fea485e91203f1eef39aa33" member_type: VOTER last_known_addr { host: "127.17.183.126" port: 38399 } } peers { permanent_uuid: "60515cb33c7343f780390d2a26bc1b94" member_type: VOTER last_known_addr { host: "127.17.183.125" port: 40657 } } peers { permanent_uuid: "f2e1a3ae51974159a8647773b8f68deb" member_type: VOTER last_known_addr { host: "127.17.183.124" port: 36315 } } }
11:40:31.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:31.312529 18399 sys_catalog.cc:458] T 00000000000000000000000000000000 P d9b357dd6fea485e91203f1eef39aa33 [sys.catalog]: This master's current role is: FOLLOWER
11:40:31.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:31.317179 18414 catalog_manager.cc:1347] Generated new cluster ID: c464bd41fe0e438bb9c4028a35e754e3
11:40:31.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:31.317440 18414 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
11:40:31.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:31.354094 18414 catalog_manager.cc:1370] Generated new certificate authority record
11:40:31.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:31.360222 18414 catalog_manager.cc:1504] Loading token signing keys...
11:40:31.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:31.390105 18414 catalog_manager.cc:5868] T 00000000000000000000000000000000 P f2e1a3ae51974159a8647773b8f68deb: Generated new TSK 0
11:40:31.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:31.392472 18414 catalog_manager.cc:1514] Initializing in-progress tserver states...
11:40:31.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:31.411188 18296 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.183.124" port: 36315 } attempt: 7
11:40:31.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:31.414580 18141 external_mini_cluster.cc:1326] Running /tmp/dist-test-task4Z5jUl/build/tsan/bin/kudu
11:40:31.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task4Z5jUl/build/tsan/bin/kudu
11:40:31.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster2916637498394586275/ts-0/wal
11:40:31.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster2916637498394586275/ts-0/data
11:40:31.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
11:40:31.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
11:40:31.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
11:40:31.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:40:31.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:40:31.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:40:31.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster2916637498394586275/ts-0/logs
11:40:31.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster2916637498394586275/ts-0/data/info.pb
11:40:31.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:40:31.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
11:40:31.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
11:40:31.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
11:40:31.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
11:40:31.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:40:31.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:40:31.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:40:31.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
11:40:31.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
11:40:31.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.183.65:0
11:40:31.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.17.183.65
11:40:31.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.183.65
11:40:31.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
11:40:31.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.17.183.126:38399,127.17.183.125:40657,127.17.183.124:36315
11:40:31.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.183.84:45103
11:40:31.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:40:31.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:40:31.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
11:40:31.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:31.467114 18296 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 60515cb33c7343f780390d2a26bc1b94: Bootstrap starting.
11:40:31.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:31.506893 18296 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 60515cb33c7343f780390d2a26bc1b94: Neither blocks nor log segments found. Creating new log.
11:40:31.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:31.518374 18296 log.cc:824] T 00000000000000000000000000000000 P 60515cb33c7343f780390d2a26bc1b94: Log is configured to *not* fsync() on all Append() calls
11:40:31.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:31.540263 18296 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 60515cb33c7343f780390d2a26bc1b94: No bootstrap required, opened a new log
11:40:31.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:31.712596 18296 raft_consensus.cc:348] T 00000000000000000000000000000000 P 60515cb33c7343f780390d2a26bc1b94 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d9b357dd6fea485e91203f1eef39aa33" member_type: VOTER last_known_addr { host: "127.17.183.126" port: 38399 } } peers { permanent_uuid: "60515cb33c7343f780390d2a26bc1b94" member_type: VOTER last_known_addr { host: "127.17.183.125" port: 40657 } } peers { permanent_uuid: "f2e1a3ae51974159a8647773b8f68deb" member_type: VOTER last_known_addr { host: "127.17.183.124" port: 36315 } }
11:40:31.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:31.717144 18296 raft_consensus.cc:374] T 00000000000000000000000000000000 P 60515cb33c7343f780390d2a26bc1b94 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
11:40:31.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:31.718575 18296 raft_consensus.cc:729] T 00000000000000000000000000000000 P 60515cb33c7343f780390d2a26bc1b94 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 60515cb33c7343f780390d2a26bc1b94, State: Initialized, Role: FOLLOWER
11:40:31.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:31.723279 18296 consensus_queue.cc:260] T 00000000000000000000000000000000 P 60515cb33c7343f780390d2a26bc1b94 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d9b357dd6fea485e91203f1eef39aa33" member_type: VOTER last_known_addr { host: "127.17.183.126" port: 38399 } } peers { permanent_uuid: "60515cb33c7343f780390d2a26bc1b94" member_type: VOTER last_known_addr { host: "127.17.183.125" port: 40657 } } peers { permanent_uuid: "f2e1a3ae51974159a8647773b8f68deb" member_type: VOTER last_known_addr { host: "127.17.183.124" port: 36315 } }
11:40:31.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:31.749878 18271 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "f2e1a3ae51974159a8647773b8f68deb" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "60515cb33c7343f780390d2a26bc1b94" is_pre_election: true
11:40:31.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:31.750666 18271 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 60515cb33c7343f780390d2a26bc1b94 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate f2e1a3ae51974159a8647773b8f68deb in term 0.
11:40:31.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:31.751919 18269 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 60515cb33c7343f780390d2a26bc1b94 [term 0 FOLLOWER]: Advancing to term 1
11:40:31.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:31.755445 18270 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "f2e1a3ae51974159a8647773b8f68deb" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "60515cb33c7343f780390d2a26bc1b94"
11:40:31.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:31.757571 18250 tablet.cc:2369] T 00000000000000000000000000000000 P 60515cb33c7343f780390d2a26bc1b94: Can't schedule compaction. Clean time has not been advanced past its initial value.
11:40:31.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:31.760814 18269 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 60515cb33c7343f780390d2a26bc1b94 [term 1 FOLLOWER]: Refusing update from remote peer f2e1a3ae51974159a8647773b8f68deb: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
11:40:31.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:31.762244 18412 consensus_queue.cc:1035] T 00000000000000000000000000000000 P f2e1a3ae51974159a8647773b8f68deb [LEADER]: Connected to new peer: Peer: permanent_uuid: "60515cb33c7343f780390d2a26bc1b94" member_type: VOTER last_known_addr { host: "127.17.183.125" port: 40657 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
11:40:31.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:31.762235 18423 sys_catalog.cc:455] T 00000000000000000000000000000000 P 60515cb33c7343f780390d2a26bc1b94 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d9b357dd6fea485e91203f1eef39aa33" member_type: VOTER last_known_addr { host: "127.17.183.126" port: 38399 } } peers { permanent_uuid: "60515cb33c7343f780390d2a26bc1b94" member_type: VOTER last_known_addr { host: "127.17.183.125" port: 40657 } } peers { permanent_uuid: "f2e1a3ae51974159a8647773b8f68deb" member_type: VOTER last_known_addr { host: "127.17.183.124" port: 36315 } } }
11:40:31.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:31.762974 18423 sys_catalog.cc:458] T 00000000000000000000000000000000 P 60515cb33c7343f780390d2a26bc1b94 [sys.catalog]: This master's current role is: FOLLOWER
11:40:31.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:31.763322 18296 sys_catalog.cc:564] T 00000000000000000000000000000000 P 60515cb33c7343f780390d2a26bc1b94 [sys.catalog]: configured and running, proceeding with master startup.
11:40:31.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:31.837699 18423 sys_catalog.cc:455] T 00000000000000000000000000000000 P 60515cb33c7343f780390d2a26bc1b94 [sys.catalog]: SysCatalogTable state changed. Reason: New leader f2e1a3ae51974159a8647773b8f68deb. Latest consensus state: current_term: 1 leader_uuid: "f2e1a3ae51974159a8647773b8f68deb" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d9b357dd6fea485e91203f1eef39aa33" member_type: VOTER last_known_addr { host: "127.17.183.126" port: 38399 } } peers { permanent_uuid: "60515cb33c7343f780390d2a26bc1b94" member_type: VOTER last_known_addr { host: "127.17.183.125" port: 40657 } } peers { permanent_uuid: "f2e1a3ae51974159a8647773b8f68deb" member_type: VOTER last_known_addr { host: "127.17.183.124" port: 36315 } } }
11:40:31.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:31.838425 18423 sys_catalog.cc:458] T 00000000000000000000000000000000 P 60515cb33c7343f780390d2a26bc1b94 [sys.catalog]: This master's current role is: FOLLOWER
11:40:31.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:31.839514 18429 sys_catalog.cc:455] T 00000000000000000000000000000000 P 60515cb33c7343f780390d2a26bc1b94 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "f2e1a3ae51974159a8647773b8f68deb" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d9b357dd6fea485e91203f1eef39aa33" member_type: VOTER last_known_addr { host: "127.17.183.126" port: 38399 } } peers { permanent_uuid: "60515cb33c7343f780390d2a26bc1b94" member_type: VOTER last_known_addr { host: "127.17.183.125" port: 40657 } } peers { permanent_uuid: "f2e1a3ae51974159a8647773b8f68deb" member_type: VOTER last_known_addr { host: "127.17.183.124" port: 36315 } } }
11:40:31.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:31.840408 18429 sys_catalog.cc:458] T 00000000000000000000000000000000 P 60515cb33c7343f780390d2a26bc1b94 [sys.catalog]: This master's current role is: FOLLOWER
11:40:31.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:31.871608 18421 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
11:40:31.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:31.872478 18421 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
11:40:31.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:31.873323 18421 flags.cc:425] Enabled unsafe flag: --never_fsync=true
11:40:31.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:31.882071 18443 catalog_manager.cc:795] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
11:40:31.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:31.927074 18421 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
11:40:31.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:31.928594 18421 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.17.183.65
11:40:31.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:31.985622 18421 tablet_server_runner.cc:78] Tablet server non-default flags:
11:40:31.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:40:31.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.183.84:45103
11:40:31.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:40:31.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:40:31.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster2916637498394586275/ts-0/data
11:40:31.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster2916637498394586275/ts-0/wal
11:40:31.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:40:31.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:40:31.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.183.65:0
11:40:31.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
11:40:31.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:40:31.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:40:31.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster2916637498394586275/ts-0/data/info.pb
11:40:31.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.183.65
11:40:31.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
11:40:31.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.17.183.126:38399,127.17.183.125:40657,127.17.183.124:36315
11:40:31.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
11:40:31.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:40:31.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
11:40:31.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
11:40:31.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:40:31.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.17.183.65
11:40:31.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster2916637498394586275/ts-0/logs
11:40:31.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:40:31.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
11:40:31.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
11:40:31.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
11:40:31.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
11:40:31.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
11:40:31.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
11:40:31.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 10:52:50 UTC on ccycloud.quasar-peqjre.root.comops.site
11:40:31.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679900
11:40:31.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
11:40:31.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:31.987133 18421 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
11:40:31.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:31.989215 18421 file_cache.cc:493] Constructed file cache file cache with capacity 419430
11:40:32.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:32.011813 18446 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:40:32.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:32.137704 18410 catalog_manager.cc:1259] Loaded cluster ID: c464bd41fe0e438bb9c4028a35e754e3
11:40:32.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:32.138151 18410 catalog_manager.cc:1552] T 00000000000000000000000000000000 P d9b357dd6fea485e91203f1eef39aa33: loading cluster ID for follower catalog manager: success
11:40:32.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:32.150061 18410 catalog_manager.cc:1574] T 00000000000000000000000000000000 P d9b357dd6fea485e91203f1eef39aa33: acquiring CA information for follower catalog manager: success
11:40:32.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:32.166194 18410 catalog_manager.cc:1602] T 00000000000000000000000000000000 P d9b357dd6fea485e91203f1eef39aa33: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
11:40:32.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:32.226723 18412 sys_catalog.cc:455] T 00000000000000000000000000000000 P f2e1a3ae51974159a8647773b8f68deb [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "f2e1a3ae51974159a8647773b8f68deb" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d9b357dd6fea485e91203f1eef39aa33" member_type: VOTER last_known_addr { host: "127.17.183.126" port: 38399 } } peers { permanent_uuid: "60515cb33c7343f780390d2a26bc1b94" member_type: VOTER last_known_addr { host: "127.17.183.125" port: 40657 } } peers { permanent_uuid: "f2e1a3ae51974159a8647773b8f68deb" member_type: VOTER last_known_addr { host: "127.17.183.124" port: 36315 } } }
11:40:32.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:32.228398 18412 sys_catalog.cc:458] T 00000000000000000000000000000000 P f2e1a3ae51974159a8647773b8f68deb [sys.catalog]: This master's current role is: LEADER
11:40:32.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:32.906780 18443 catalog_manager.cc:1259] Loaded cluster ID: c464bd41fe0e438bb9c4028a35e754e3
11:40:32.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:32.907305 18443 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 60515cb33c7343f780390d2a26bc1b94: loading cluster ID for follower catalog manager: success
11:40:32.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:32.921245 18443 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 60515cb33c7343f780390d2a26bc1b94: acquiring CA information for follower catalog manager: success
11:40:32.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:32.951836 18443 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 60515cb33c7343f780390d2a26bc1b94: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
11:40:33.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:32.012495 18445 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:40:33.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:33.406347 18444 debug-util.cc:398] Leaking SignalData structure 0x7b0800014860 after lost signal to thread 18421
11:40:33.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:33.631039 18444 kernel_stack_watchdog.cc:198] Thread 18421 stuck at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/thread.cc:637 for 390ms:
11:40:33.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
11:40:33.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
11:40:33.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
11:40:33.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
11:40:33.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:33.635517 18447 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1621 milliseconds
11:40:33.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:33.636345 18421 thread.cc:636] OpenStack (cloud detector) Time spent creating pthread: real 1.623s user 0.001s sys 0.001s
11:40:33.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:33.636729 18421 thread.cc:602] OpenStack (cloud detector) Time spent starting thread: real 1.623s user 0.001s sys 0.001s
11:40:33.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:33.637567 18448 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:40:33.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:33.637594 18421 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
11:40:33.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:33.639041 18421 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
11:40:33.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:33.651566 18421 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
11:40:33.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:33.653045 18421 hybrid_clock.cc:648] HybridClock initialized: now 1790682033653003 us; error 41 us; skew 500 ppm
11:40:33.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:33.654119 18421 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
11:40:33.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:33.684686 18421 webserver.cc:460] Webserver started at http://127.17.183.65:46025/ using document root <none> and password file <none>
11:40:33.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:33.686764 18421 fs_manager.cc:359] Metadata directory not provided
11:40:33.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:33.687094 18421 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
11:40:33.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:33.687673 18421 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
11:40:33.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:33.698434 18421 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster2916637498394586275/ts-0/data/instance:
11:40:33.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2b135379fccd4f9191f7dd7db60a8f82"
11:40:33.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 11:40:33 on dist-test-slave-h1fj"
11:40:33.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:33.699863 18421 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster2916637498394586275/ts-0/wal/instance:
11:40:33.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2b135379fccd4f9191f7dd7db60a8f82"
11:40:33.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 11:40:33 on dist-test-slave-h1fj"
11:40:33.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:33.716228 18421 fs_manager.cc:691] Time spent creating directory manager: real 0.016s user 0.010s sys 0.004s
11:40:33.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:33.729110 18460 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
11:40:33.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:33.730252 18421 fs_manager.cc:725] Time spent opening block manager: real 0.010s user 0.002s sys 0.007s
11:40:33.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:33.730597 18421 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster2916637498394586275/ts-0/data,/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster2916637498394586275/ts-0/wal
11:40:33.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2b135379fccd4f9191f7dd7db60a8f82"
11:40:33.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 11:40:33 on dist-test-slave-h1fj"
11:40:33.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:33.731004 18421 fs_report.cc:383] FS layout report
11:40:33.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
11:40:33.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster2916637498394586275/ts-0/wal
11:40:33.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster2916637498394586275/ts-0/wal
11:40:33.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster2916637498394586275/ts-0/data/data
11:40:33.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
11:40:33.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
11:40:33.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
11:40:33.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
11:40:33.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
11:40:33.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
11:40:33.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
11:40:33.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
11:40:33.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
11:40:33.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
11:40:33.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
11:40:33.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:33.803349 18421 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
11:40:33.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:33.805223 18421 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
11:40:33.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:33.805778 18421 kserver.cc:163] Server-wide thread pool size limit: 3276
11:40:33.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:33.815171 18421 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
11:40:33.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:33.827512 18421 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
11:40:33.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:33.827769 18421 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
11:40:33.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:33.828049 18421 ts_tablet_manager.cc:610] Registered 0 tablets
11:40:33.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:33.828243 18421 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
11:40:34.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:34.296341 18421 rpc_server.cc:307] RPC server started. Bound to: 127.17.183.65:38327
11:40:34.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:34.296475 18572 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.17.183.65:38327 every 8 connection(s)
11:40:34.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:34.298976 18421 server_base.cc:1166] Dumped server information to /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster2916637498394586275/ts-0/data/info.pb
11:40:34.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:34.301956 18141 external_mini_cluster.cc:1388] Started /tmp/dist-test-task4Z5jUl/build/tsan/bin/kudu as pid 18421
11:40:34.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:34.302429 18141 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster2916637498394586275/ts-0/wal/instance
11:40:34.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:34.314335 18141 external_mini_cluster.cc:1326] Running /tmp/dist-test-task4Z5jUl/build/tsan/bin/kudu
11:40:34.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task4Z5jUl/build/tsan/bin/kudu
11:40:34.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster2916637498394586275/ts-1/wal
11:40:34.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster2916637498394586275/ts-1/data
11:40:34.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
11:40:34.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
11:40:34.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
11:40:34.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:40:34.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:40:34.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:40:34.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster2916637498394586275/ts-1/logs
11:40:34.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster2916637498394586275/ts-1/data/info.pb
11:40:34.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:40:34.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
11:40:34.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
11:40:34.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
11:40:34.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
11:40:34.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:40:34.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:40:34.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:40:34.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
11:40:34.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
11:40:34.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.183.66:0
11:40:34.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.17.183.66
11:40:34.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.183.66
11:40:34.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
11:40:34.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.17.183.126:38399,127.17.183.125:40657,127.17.183.124:36315
11:40:34.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.183.84:45103
11:40:34.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:40:34.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:40:34.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
11:40:34.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:34.388379 18573 heartbeater.cc:347] Connected to a master server at 127.17.183.124:36315
11:40:34.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:34.388854 18573 heartbeater.cc:464] Registering TS with master...
11:40:34.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:34.389962 18573 heartbeater.cc:511] Master 127.17.183.124:36315 requested a full tablet report, sending...
11:40:34.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:34.393106 18340 ts_manager.cc:194] Registered new tserver with Master: 2b135379fccd4f9191f7dd7db60a8f82 (127.17.183.65:38327)
11:40:34.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:34.395840 18340 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.17.183.65:42505
11:40:34.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:34.398851 18574 heartbeater.cc:347] Connected to a master server at 127.17.183.125:40657
11:40:34.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:34.399191 18574 heartbeater.cc:464] Registering TS with master...
11:40:34.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:34.400730 18574 heartbeater.cc:511] Master 127.17.183.125:40657 requested a full tablet report, sending...
11:40:34.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:34.403903 18261 ts_manager.cc:194] Registered new tserver with Master: 2b135379fccd4f9191f7dd7db60a8f82 (127.17.183.65:38327)
11:40:34.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:34.413089 18575 heartbeater.cc:347] Connected to a master server at 127.17.183.126:38399
11:40:34.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:34.413406 18575 heartbeater.cc:464] Registering TS with master...
11:40:34.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:34.414079 18575 heartbeater.cc:511] Master 127.17.183.126:38399 requested a full tablet report, sending...
11:40:34.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:34.417016 18185 ts_manager.cc:194] Registered new tserver with Master: 2b135379fccd4f9191f7dd7db60a8f82 (127.17.183.65:38327)
11:40:34.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:34.673002 18580 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
11:40:34.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:34.673430 18580 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
11:40:34.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:34.673872 18580 flags.cc:425] Enabled unsafe flag: --never_fsync=true
11:40:34.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:34.720615 18580 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
11:40:34.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:34.721421 18580 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.17.183.66
11:40:34.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:34.754422 18580 tablet_server_runner.cc:78] Tablet server non-default flags:
11:40:34.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:40:34.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.183.84:45103
11:40:34.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:40:34.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:40:34.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster2916637498394586275/ts-1/data
11:40:34.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster2916637498394586275/ts-1/wal
11:40:34.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:40:34.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:40:34.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.183.66:0
11:40:34.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
11:40:34.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:40:34.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:40:34.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster2916637498394586275/ts-1/data/info.pb
11:40:34.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.183.66
11:40:34.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
11:40:34.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.17.183.126:38399,127.17.183.125:40657,127.17.183.124:36315
11:40:34.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
11:40:34.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:40:34.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
11:40:34.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
11:40:34.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:40:34.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.17.183.66
11:40:34.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster2916637498394586275/ts-1/logs
11:40:34.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:40:34.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
11:40:34.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
11:40:34.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
11:40:34.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
11:40:34.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
11:40:34.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
11:40:34.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 10:52:50 UTC on ccycloud.quasar-peqjre.root.comops.site
11:40:34.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679900
11:40:34.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
11:40:34.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:34.755525 18580 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
11:40:34.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:34.756975 18580 file_cache.cc:493] Constructed file cache file cache with capacity 419430
11:40:34.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:34.776571 18590 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:40:35.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:35.406388 18573 heartbeater.cc:503] Master 127.17.183.124:36315 was elected leader, sending a full tablet report...
11:40:35.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:34.776813 18592 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:40:35.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:34.777251 18589 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:40:35.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:34.779691 18580 server_base.cc:1034] running on GCE node
11:40:35.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:35.882277 18580 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
11:40:35.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:35.884503 18580 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
11:40:35.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:35.885844 18580 hybrid_clock.cc:648] HybridClock initialized: now 1790682035885803 us; error 40 us; skew 500 ppm
11:40:35.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:35.886618 18580 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
11:40:35.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:35.892380 18580 webserver.cc:460] Webserver started at http://127.17.183.66:39581/ using document root <none> and password file <none>
11:40:35.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:35.893244 18580 fs_manager.cc:359] Metadata directory not provided
11:40:35.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:35.893445 18580 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
11:40:35.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:35.893889 18580 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
11:40:35.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:35.898846 18580 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster2916637498394586275/ts-1/data/instance:
11:40:35.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "294de7e59e394ee2a70c71b2c73e533d"
11:40:35.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 11:40:35 on dist-test-slave-h1fj"
11:40:35.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:35.899923 18580 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster2916637498394586275/ts-1/wal/instance:
11:40:35.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "294de7e59e394ee2a70c71b2c73e533d"
11:40:35.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 11:40:35 on dist-test-slave-h1fj"
11:40:35.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:35.906653 18580 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.002s sys 0.004s
11:40:35.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:35.911923 18600 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
11:40:35.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:35.912804 18580 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.002s
11:40:35.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:35.913105 18580 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster2916637498394586275/ts-1/data,/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster2916637498394586275/ts-1/wal
11:40:35.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "294de7e59e394ee2a70c71b2c73e533d"
11:40:35.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 11:40:35 on dist-test-slave-h1fj"
11:40:35.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:35.913398 18580 fs_report.cc:383] FS layout report
11:40:35.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
11:40:35.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster2916637498394586275/ts-1/wal
11:40:35.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster2916637498394586275/ts-1/wal
11:40:35.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster2916637498394586275/ts-1/data/data
11:40:35.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
11:40:35.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
11:40:35.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
11:40:35.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
11:40:35.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
11:40:35.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
11:40:35.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
11:40:35.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
11:40:35.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
11:40:35.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
11:40:35.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
11:40:35.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:35.944725 18580 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
11:40:35.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:35.946130 18580 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
11:40:35.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:35.946564 18580 kserver.cc:163] Server-wide thread pool size limit: 3276
11:40:35.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:35.948920 18580 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
11:40:35.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:35.952874 18580 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
11:40:35.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:35.953094 18580 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
11:40:35.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:35.953374 18580 ts_tablet_manager.cc:610] Registered 0 tablets
11:40:35.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:35.953538 18580 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
11:40:36.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:36.080168 18580 rpc_server.cc:307] RPC server started. Bound to: 127.17.183.66:41569
11:40:36.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:36.080235 18712 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.17.183.66:41569 every 8 connection(s)
11:40:36.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:36.083597 18580 server_base.cc:1166] Dumped server information to /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster2916637498394586275/ts-1/data/info.pb
11:40:36.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:36.089454 18141 external_mini_cluster.cc:1388] Started /tmp/dist-test-task4Z5jUl/build/tsan/bin/kudu as pid 18580
11:40:36.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:36.090106 18141 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster2916637498394586275/ts-1/wal/instance
11:40:36.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:36.106897 18141 external_mini_cluster.cc:1326] Running /tmp/dist-test-task4Z5jUl/build/tsan/bin/kudu
11:40:36.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task4Z5jUl/build/tsan/bin/kudu
11:40:36.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster2916637498394586275/ts-2/wal
11:40:36.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster2916637498394586275/ts-2/data
11:40:36.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
11:40:36.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
11:40:36.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
11:40:36.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:40:36.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:40:36.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:40:36.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster2916637498394586275/ts-2/logs
11:40:36.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster2916637498394586275/ts-2/data/info.pb
11:40:36.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:40:36.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
11:40:36.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
11:40:36.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
11:40:36.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
11:40:36.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:40:36.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:40:36.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:40:36.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
11:40:36.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
11:40:36.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.183.67:0
11:40:36.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.17.183.67
11:40:36.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.183.67
11:40:36.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
11:40:36.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.17.183.126:38399,127.17.183.125:40657,127.17.183.124:36315
11:40:36.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.183.84:45103
11:40:36.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:40:36.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:40:36.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
11:40:36.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:36.154511 18713 heartbeater.cc:347] Connected to a master server at 127.17.183.124:36315
11:40:36.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:36.154810 18714 heartbeater.cc:347] Connected to a master server at 127.17.183.125:40657
11:40:36.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:36.154964 18713 heartbeater.cc:464] Registering TS with master...
11:40:36.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:36.155153 18714 heartbeater.cc:464] Registering TS with master...
11:40:36.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:36.156370 18713 heartbeater.cc:511] Master 127.17.183.124:36315 requested a full tablet report, sending...
11:40:36.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:36.156472 18714 heartbeater.cc:511] Master 127.17.183.125:40657 requested a full tablet report, sending...
11:40:36.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:36.158826 18340 ts_manager.cc:194] Registered new tserver with Master: 294de7e59e394ee2a70c71b2c73e533d (127.17.183.66:41569)
11:40:36.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:36.159837 18261 ts_manager.cc:194] Registered new tserver with Master: 294de7e59e394ee2a70c71b2c73e533d (127.17.183.66:41569)
11:40:36.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:36.160715 18340 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.17.183.66:50075
11:40:36.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:36.173663 18715 heartbeater.cc:347] Connected to a master server at 127.17.183.126:38399
11:40:36.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:36.173938 18715 heartbeater.cc:464] Registering TS with master...
11:40:36.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:36.174497 18715 heartbeater.cc:511] Master 127.17.183.126:38399 requested a full tablet report, sending...
11:40:36.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:36.176858 18185 ts_manager.cc:194] Registered new tserver with Master: 294de7e59e394ee2a70c71b2c73e533d (127.17.183.66:41569)
11:40:36.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:36.433568 18723 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
11:40:36.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:36.434060 18723 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
11:40:36.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:36.434572 18723 flags.cc:425] Enabled unsafe flag: --never_fsync=true
11:40:36.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:36.464699 18723 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
11:40:36.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:36.465575 18723 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.17.183.67
11:40:36.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:36.499106 18723 tablet_server_runner.cc:78] Tablet server non-default flags:
11:40:36.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:40:36.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.183.84:45103
11:40:36.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:40:36.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:40:36.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster2916637498394586275/ts-2/data
11:40:36.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster2916637498394586275/ts-2/wal
11:40:36.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:40:36.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:40:36.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.183.67:0
11:40:36.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
11:40:36.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:40:36.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:40:36.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster2916637498394586275/ts-2/data/info.pb
11:40:36.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.183.67
11:40:36.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
11:40:36.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.17.183.126:38399,127.17.183.125:40657,127.17.183.124:36315
11:40:36.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
11:40:36.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:40:36.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
11:40:36.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
11:40:36.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:40:36.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.17.183.67
11:40:36.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster2916637498394586275/ts-2/logs
11:40:36.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:40:36.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
11:40:36.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
11:40:36.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
11:40:36.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
11:40:36.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
11:40:36.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
11:40:36.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 10:52:50 UTC on ccycloud.quasar-peqjre.root.comops.site
11:40:36.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679900
11:40:36.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
11:40:36.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:36.500496 18723 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
11:40:36.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:36.501966 18723 file_cache.cc:493] Constructed file cache file cache with capacity 419430
11:40:36.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:36.518973 18730 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:40:37.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:37.167470 18713 heartbeater.cc:503] Master 127.17.183.124:36315 was elected leader, sending a full tablet report...
11:40:37.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:36.518999 18733 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:40:37.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:36.519155 18731 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:40:37.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:36.520349 18723 server_base.cc:1034] running on GCE node
11:40:37.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:37.584872 18723 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
11:40:37.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:37.587081 18723 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
11:40:37.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:37.588498 18723 hybrid_clock.cc:648] HybridClock initialized: now 1790682037588450 us; error 53 us; skew 500 ppm
11:40:37.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:37.589288 18723 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
11:40:37.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:37.595005 18723 webserver.cc:460] Webserver started at http://127.17.183.67:33377/ using document root <none> and password file <none>
11:40:37.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:37.595964 18723 fs_manager.cc:359] Metadata directory not provided
11:40:37.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:37.596168 18723 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
11:40:37.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:37.596654 18723 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
11:40:37.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:37.601293 18723 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster2916637498394586275/ts-2/data/instance:
11:40:37.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6ae93c5e6d9842d9be303a3d17bf3718"
11:40:37.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 11:40:37 on dist-test-slave-h1fj"
11:40:37.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:37.602341 18723 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster2916637498394586275/ts-2/wal/instance:
11:40:37.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6ae93c5e6d9842d9be303a3d17bf3718"
11:40:37.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 11:40:37 on dist-test-slave-h1fj"
11:40:37.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:37.609028 18723 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.008s sys 0.000s
11:40:37.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:37.614092 18740 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
11:40:37.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:37.615015 18723 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.002s sys 0.000s
11:40:37.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:37.615329 18723 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster2916637498394586275/ts-2/data,/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster2916637498394586275/ts-2/wal
11:40:37.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6ae93c5e6d9842d9be303a3d17bf3718"
11:40:37.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 11:40:37 on dist-test-slave-h1fj"
11:40:37.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:37.615630 18723 fs_report.cc:383] FS layout report
11:40:37.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
11:40:37.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster2916637498394586275/ts-2/wal
11:40:37.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster2916637498394586275/ts-2/wal
11:40:37.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster2916637498394586275/ts-2/data/data
11:40:37.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
11:40:37.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
11:40:37.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
11:40:37.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
11:40:37.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
11:40:37.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
11:40:37.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
11:40:37.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
11:40:37.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
11:40:37.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
11:40:37.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
11:40:37.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:37.659605 18723 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
11:40:37.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:37.661113 18723 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
11:40:37.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:37.661549 18723 kserver.cc:163] Server-wide thread pool size limit: 3276
11:40:37.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:37.664052 18723 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
11:40:37.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:37.668236 18723 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
11:40:37.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:37.668442 18723 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
11:40:37.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:37.668722 18723 ts_tablet_manager.cc:610] Registered 0 tablets
11:40:37.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:37.668885 18723 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
11:40:37.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:37.804919 18723 rpc_server.cc:307] RPC server started. Bound to: 127.17.183.67:37389
11:40:37.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:37.805020 18853 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.17.183.67:37389 every 8 connection(s)
11:40:37.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:37.807431 18723 server_base.cc:1166] Dumped server information to /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster2916637498394586275/ts-2/data/info.pb
11:40:37.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:37.810374 18141 external_mini_cluster.cc:1388] Started /tmp/dist-test-task4Z5jUl/build/tsan/bin/kudu as pid 18723
11:40:37.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:37.810796 18141 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster2916637498394586275/ts-2/wal/instance
11:40:37.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:37.852505 18855 heartbeater.cc:347] Connected to a master server at 127.17.183.125:40657
11:40:37.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:37.852803 18854 heartbeater.cc:347] Connected to a master server at 127.17.183.124:36315
11:40:37.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:37.852950 18855 heartbeater.cc:464] Registering TS with master...
11:40:37.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:37.853106 18854 heartbeater.cc:464] Registering TS with master...
11:40:37.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:37.854123 18856 heartbeater.cc:347] Connected to a master server at 127.17.183.126:38399
11:40:37.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:37.854214 18854 heartbeater.cc:511] Master 127.17.183.124:36315 requested a full tablet report, sending...
11:40:37.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:37.854372 18856 heartbeater.cc:464] Registering TS with master...
11:40:37.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:37.854959 18856 heartbeater.cc:511] Master 127.17.183.126:38399 requested a full tablet report, sending...
11:40:37.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:37.855780 18855 heartbeater.cc:511] Master 127.17.183.125:40657 requested a full tablet report, sending...
11:40:37.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:37.856447 18340 ts_manager.cc:194] Registered new tserver with Master: 6ae93c5e6d9842d9be303a3d17bf3718 (127.17.183.67:37389)
11:40:37.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:37.857084 18185 ts_manager.cc:194] Registered new tserver with Master: 6ae93c5e6d9842d9be303a3d17bf3718 (127.17.183.67:37389)
11:40:37.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:37.857913 18340 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.17.183.67:52913
11:40:37.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:37.860620 18261 ts_manager.cc:194] Registered new tserver with Master: 6ae93c5e6d9842d9be303a3d17bf3718 (127.17.183.67:37389)
11:40:37.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:37.870510 18141 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
11:40:37.880 [DEBUG - main] (MiniKuduCluster.java:195) Response:
11:40:37.880 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
11:40:37.881 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.17.183.126"
port: 38399
}
bound_http_address {
host: "127.17.183.126"
port: 43951
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.17.183.125"
port: 40657
}
bound_http_address {
host: "127.17.183.125"
port: 45007
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.17.183.124"
port: 36315
}
bound_http_address {
host: "127.17.183.124"
port: 38717
}
}
}
11:40:37.881 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
11:40:37.882 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.17.183.65"
port: 38327
}
bound_http_address {
host: "127.17.183.65"
port: 46025
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.17.183.66"
port: 41569
}
bound_http_address {
host: "127.17.183.66"
port: 39581
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.17.183.67"
port: 37389
}
bound_http_address {
host: "127.17.183.67"
port: 33377
}
}
}
11:40:37.882 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
11:40:37.883 [DEBUG - main] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster6138638037486277202/krb5kdc/krb5cc
11:40:37.883 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
11:40:37.884 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-task4Z5jUl/test-tmp
11:40:37.885 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
11:40:37.885 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-task4Z5jUl/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
11:40:37.886 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213"
extra_master_flags: "--authn_token_validity_seconds=10"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
11:40:38.205 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
11:40:38.205 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
11:40:38.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:38.208210 18865 mini_oidc.cc:152] Starting JWKS server
11:40:38.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:38.211354 18865 webserver.cc:460] Webserver started at http://127.0.0.1:45823/ using document root <none> and password file <none>
11:40:38.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:38.211743 18865 mini_oidc.cc:189] Starting OIDC Discovery server
11:40:38.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:38.213263 18865 webserver.cc:460] Webserver started at http://127.0.0.1:43773/ using document root <none> and password file <none>
11:40:38.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
11:40:38.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/krb5kdc/principal' for realm 'KRBTEST.COM',
11:40:38.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
11:40:38.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 11:40:38 dist-test-slave-h1fj krb5kdc[18876](info): setting up network...
11:40:38.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
11:40:38.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 11:40:38 dist-test-slave-h1fj krb5kdc[18876](info): set up 2 sockets
11:40:38.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 11:40:38 dist-test-slave-h1fj krb5kdc[18876](info): commencing operation
11:40:38.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
11:40:38.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:38.861709 18854 heartbeater.cc:503] Master 127.17.183.124:36315 was elected leader, sending a full tablet report...
11:40:39.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:39.202523 18865 mini_kdc.cc:121] Time spent starting KDC: real 0.967s user 0.005s sys 0.011s
11:40:39.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
11:40:39.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
11:40:39.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
11:40:39.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
11:40:39.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
11:40:39.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
11:40:39.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
11:40:39.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
11:40:39.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
11:40:39.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
11:40:39.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/krb5kdc/test-user.keytab.
11:40:39.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/krb5kdc/test-user.keytab.
11:40:39.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 11:40:39 dist-test-slave-h1fj krb5kdc[18876](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790682039, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
11:40:39.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
11:40:39.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-29T11:40:39Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
11:40:39.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-29T11:40:39Z Disabled control of system clock
11:40:39.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.18.108.126@KRBTEST.COM; defaulting to no policy
11:40:39.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
11:40:39.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.18.108.126@KRBTEST.COM" created.
11:40:39.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
11:40:39.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.18.108.126 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/krb5kdc/kudu_127.18.108.126.keytab.
11:40:39.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.18.108.126 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/krb5kdc/kudu_127.18.108.126.keytab.
11:40:39.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:39.372169 18865 external_mini_cluster.cc:1326] Running /tmp/dist-test-task4Z5jUl/build/tsan/bin/kudu
11:40:39.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task4Z5jUl/build/tsan/bin/kudu
11:40:39.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-0/wal
11:40:39.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-0/data
11:40:39.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
11:40:39.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
11:40:39.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
11:40:39.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:40:39.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:40:39.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:40:39.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-0/logs
11:40:39.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-0/data/info.pb
11:40:39.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:40:39.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
11:40:39.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
11:40:39.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
11:40:39.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
11:40:39.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:40:39.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:40:39.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:40:39.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
11:40:39.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
11:40:39.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
11:40:39.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
11:40:39.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.108.126:43211
11:40:39.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.108.126
11:40:39.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
11:40:39.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/krb5kdc/kudu_127.18.108.126.keytab
11:40:39.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.18.108.126
11:40:39.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
11:40:39.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
11:40:39.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
11:40:39.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.108.84:34171
11:40:39.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:40:39.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:40:39.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:40:39.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
11:40:39.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.18.108.126:43211,127.18.108.125:36939,127.18.108.124:42121
11:40:39.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
11:40:39.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45823/jwks.json
11:40:39.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
11:40:39.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:39.660384 18891 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
11:40:39.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:39.660866 18891 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
11:40:39.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:39.661437 18891 flags.cc:425] Enabled unsafe flag: --never_fsync=true
11:40:39.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:39.699474 18891 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
11:40:39.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:39.699769 18891 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
11:40:39.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:39.700187 18891 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
11:40:39.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:39.700495 18891 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
11:40:39.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:39.700784 18891 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
11:40:39.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:39.701072 18891 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
11:40:39.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:39.701298 18891 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:45823/jwks.json
11:40:39.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:39.741405 18891 master_runner.cc:386] Master server non-default flags:
11:40:39.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:40:39.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.108.84:34171
11:40:39.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:40:39.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:40:39.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-0/data
11:40:39.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-0/wal
11:40:39.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
11:40:39.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
11:40:39.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.18.108.126:43211,127.18.108.125:36939,127.18.108.124:42121
11:40:39.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/krb5kdc/kudu_127.18.108.126.keytab
11:40:39.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.18.108.126
11:40:39.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:40:39.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:40:39.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
11:40:39.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.108.126:43211
11:40:39.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
11:40:39.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
11:40:39.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
11:40:39.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45823/jwks.json
11:40:39.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
11:40:39.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
11:40:39.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
11:40:39.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:40:39.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:40:39.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-0/data/info.pb
11:40:39.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.108.126
11:40:39.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
11:40:39.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
11:40:39.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:40:39.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
11:40:39.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
11:40:39.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:40:39.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-0/logs
11:40:39.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:40:39.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
11:40:39.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
11:40:39.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
11:40:39.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
11:40:39.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
11:40:39.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
11:40:39.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 10:52:50 UTC on ccycloud.quasar-peqjre.root.comops.site
11:40:39.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679900
11:40:39.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
11:40:39.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:39.742489 18891 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
11:40:39.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:39.743469 18891 file_cache.cc:493] Constructed file cache file cache with capacity 419430
11:40:39.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:39.759109 18897 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:40:40.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:39.759124 18900 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:40:40.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:39.759852 18898 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:40:40.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:39.761425 18891 server_base.cc:1034] running on GCE node
11:40:40.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:40.867779 18891 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
11:40:40.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:40.870821 18891 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
11:40:40.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:40.872226 18891 hybrid_clock.cc:648] HybridClock initialized: now 1790682040872184 us; error 40 us; skew 500 ppm
11:40:40.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:40.873019 18891 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
11:40:40.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 11:40:40 dist-test-slave-h1fj krb5kdc[18876](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790682040, etypes {rep=17 tkt=17 ses=17}, kudu/127.18.108.126@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
11:40:40.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:40.882050 18891 init.cc:377] Logged in from keytab as kudu/127.18.108.126@KRBTEST.COM (short username kudu)
11:40:40.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:40.886817 18891 webserver.cc:460] Webserver started at http://127.18.108.126:36579/ using document root <none> and password file <none>
11:40:40.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:40.887689 18891 fs_manager.cc:359] Metadata directory not provided
11:40:40.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:40.887861 18891 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
11:40:40.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:40.888276 18891 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
11:40:40.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:40.893539 18891 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-0/data/instance:
11:40:40.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "eb8f4e2d825e4e319fd704623a2750bf"
11:40:40.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 11:40:40 on dist-test-slave-h1fj"
11:40:40.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:40.894629 18891 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-0/wal/instance:
11:40:40.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "eb8f4e2d825e4e319fd704623a2750bf"
11:40:40.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 11:40:40 on dist-test-slave-h1fj"
11:40:40.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:40.901366 18891 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.009s sys 0.000s
11:40:40.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:40.906925 18908 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
11:40:40.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:40.907936 18891 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
11:40:40.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:40.908190 18891 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-0/data,/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-0/wal
11:40:40.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "eb8f4e2d825e4e319fd704623a2750bf"
11:40:40.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 11:40:40 on dist-test-slave-h1fj"
11:40:40.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:40.908515 18891 fs_report.cc:383] FS layout report
11:40:40.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
11:40:40.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-0/wal
11:40:40.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-0/wal
11:40:40.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-0/data/data
11:40:40.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
11:40:40.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
11:40:40.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
11:40:40.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
11:40:40.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
11:40:40.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
11:40:40.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
11:40:40.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
11:40:40.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
11:40:40.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
11:40:40.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
11:40:40.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:40.956566 18891 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
11:40:40.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:40.961773 18891 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
11:40:40.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:40.962153 18891 kserver.cc:163] Server-wide thread pool size limit: 3276
11:40:41.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:41.026458 18891 rpc_server.cc:307] RPC server started. Bound to: 127.18.108.126:43211
11:40:41.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:41.026531 18959 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.18.108.126:43211 every 8 connection(s)
11:40:41.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:41.029104 18891 server_base.cc:1166] Dumped server information to /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-0/data/info.pb
11:40:41.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:41.034682 18865 external_mini_cluster.cc:1388] Started /tmp/dist-test-task4Z5jUl/build/tsan/bin/kudu as pid 18891
11:40:41.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:41.034863 18960 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
11:40:41.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:41.035118 18865 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-0/wal/instance
11:40:41.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:41.043314 18850 debug-util.cc:398] Leaking SignalData structure 0x7b08000bb380 after lost signal to thread 18725
11:40:41.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:41.044376 18850 debug-util.cc:398] Leaking SignalData structure 0x7b08000afd20 after lost signal to thread 18853
11:40:41.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.18.108.125@KRBTEST.COM; defaulting to no policy
11:40:41.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:41.046185 18960 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.18.108.126" port: 43211 } has no permanent_uuid. Determining permanent_uuid...
11:40:41.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
11:40:41.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.18.108.125@KRBTEST.COM" created.
11:40:41.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 11:40:41 dist-test-slave-h1fj krb5kdc[18876](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790682040, etypes {rep=17 tkt=17 ses=17}, kudu/127.18.108.126@KRBTEST.COM for kudu/127.18.108.126@KRBTEST.COM
11:40:41.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
11:40:41.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.18.108.125 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/krb5kdc/kudu_127.18.108.125.keytab.
11:40:41.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.18.108.125 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/krb5kdc/kudu_127.18.108.125.keytab.
11:40:41.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:41.080714 18865 external_mini_cluster.cc:1326] Running /tmp/dist-test-task4Z5jUl/build/tsan/bin/kudu
11:40:41.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task4Z5jUl/build/tsan/bin/kudu
11:40:41.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-1/wal
11:40:41.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-1/data
11:40:41.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
11:40:41.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
11:40:41.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
11:40:41.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:40:41.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:40:41.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:40:41.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-1/logs
11:40:41.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-1/data/info.pb
11:40:41.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:40:41.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
11:40:41.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
11:40:41.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
11:40:41.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
11:40:41.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:40:41.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:40:41.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:40:41.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
11:40:41.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
11:40:41.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
11:40:41.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
11:40:41.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.108.125:36939
11:40:41.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.108.125
11:40:41.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
11:40:41.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/krb5kdc/kudu_127.18.108.125.keytab
11:40:41.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.18.108.125
11:40:41.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
11:40:41.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
11:40:41.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
11:40:41.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.108.84:34171
11:40:41.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:40:41.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:40:41.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:40:41.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
11:40:41.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.18.108.126:43211,127.18.108.125:36939,127.18.108.124:42121
11:40:41.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
11:40:41.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45823/jwks.json
11:40:41.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
11:40:41.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:41.101740 18960 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.18.108.125" port: 36939 } has no permanent_uuid. Determining permanent_uuid...
11:40:41.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:41.103770 18909 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.18.108.125:36939: connect: Connection refused (error 111)
11:40:41.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:41.111068 18960 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.18.108.125:36939: Network error: Client connection negotiation failed: client connection to 127.18.108.125:36939: connect: Connection refused (error 111)
11:40:41.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:41.160539 18960 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.108.125" port: 36939 } attempt: 1
11:40:41.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:41.164162 18960 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.18.108.125:36939: Network error: Client connection negotiation failed: client connection to 127.18.108.125:36939: connect: Connection refused (error 111)
11:40:41.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:41.232543 18960 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.108.125" port: 36939 } attempt: 2
11:40:41.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:41.235805 18960 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.18.108.125:36939: Network error: Client connection negotiation failed: client connection to 127.18.108.125:36939: connect: Connection refused (error 111)
11:40:41.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:41.327138 18960 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.108.125" port: 36939 } attempt: 3
11:40:41.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:41.331171 18960 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.18.108.125:36939: Network error: Client connection negotiation failed: client connection to 127.18.108.125:36939: connect: Connection refused (error 111)
11:40:41.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:41.474582 18960 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.108.125" port: 36939 } attempt: 4
11:40:41.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:41.479094 18960 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.18.108.125:36939: Network error: Client connection negotiation failed: client connection to 127.18.108.125:36939: connect: Connection refused (error 111)
11:40:41.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:41.487834 18966 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
11:40:41.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:41.488375 18966 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
11:40:41.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:41.489039 18966 flags.cc:425] Enabled unsafe flag: --never_fsync=true
11:40:41.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:41.519948 18966 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
11:40:41.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:41.520151 18966 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
11:40:41.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:41.520408 18966 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
11:40:41.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:41.520596 18966 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
11:40:41.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:41.520776 18966 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
11:40:41.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:41.520964 18966 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
11:40:41.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:41.521096 18966 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:45823/jwks.json
11:40:41.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:41.556092 18966 master_runner.cc:386] Master server non-default flags:
11:40:41.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:40:41.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.108.84:34171
11:40:41.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:40:41.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:40:41.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-1/data
11:40:41.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-1/wal
11:40:41.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
11:40:41.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
11:40:41.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.18.108.126:43211,127.18.108.125:36939,127.18.108.124:42121
11:40:41.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/krb5kdc/kudu_127.18.108.125.keytab
11:40:41.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.18.108.125
11:40:41.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:40:41.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:40:41.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
11:40:41.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.108.125:36939
11:40:41.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
11:40:41.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
11:40:41.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
11:40:41.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45823/jwks.json
11:40:41.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
11:40:41.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
11:40:41.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
11:40:41.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:40:41.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:40:41.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-1/data/info.pb
11:40:41.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.108.125
11:40:41.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
11:40:41.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
11:40:41.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:40:41.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
11:40:41.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
11:40:41.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:40:41.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-1/logs
11:40:41.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:40:41.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
11:40:41.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
11:40:41.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
11:40:41.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
11:40:41.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
11:40:41.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
11:40:41.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 10:52:50 UTC on ccycloud.quasar-peqjre.root.comops.site
11:40:41.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679900
11:40:41.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
11:40:41.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:41.557123 18966 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
11:40:41.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:41.558151 18966 file_cache.cc:493] Constructed file cache file cache with capacity 419430
11:40:41.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:41.576061 18976 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:40:41.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:41.778505 18960 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.108.125" port: 36939 } attempt: 5
11:40:41.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:41.782603 18960 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.18.108.125:36939: Network error: Client connection negotiation failed: client connection to 127.18.108.125:36939: connect: Connection refused (error 111)
11:40:42.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:42.329995 18960 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.108.125" port: 36939 } attempt: 6
11:40:42.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:42.337229 18960 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.18.108.125:36939: Network error: Client connection negotiation failed: client connection to 127.18.108.125:36939: connect: Connection refused (error 111)
11:40:42.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:41.576074 18974 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:40:42.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:41.577461 18966 server_base.cc:1034] running on GCE node
11:40:42.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:41.577384 18973 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:40:42.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:42.704394 18966 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
11:40:42.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:42.707377 18966 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
11:40:42.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:42.708796 18966 hybrid_clock.cc:648] HybridClock initialized: now 1790682042708712 us; error 79 us; skew 500 ppm
11:40:42.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:42.709551 18966 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
11:40:42.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 11:40:42 dist-test-slave-h1fj krb5kdc[18876](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790682042, etypes {rep=17 tkt=17 ses=17}, kudu/127.18.108.125@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
11:40:42.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:42.714867 18966 init.cc:377] Logged in from keytab as kudu/127.18.108.125@KRBTEST.COM (short username kudu)
11:40:42.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:42.719682 18966 webserver.cc:460] Webserver started at http://127.18.108.125:38867/ using document root <none> and password file <none>
11:40:42.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:42.720487 18966 fs_manager.cc:359] Metadata directory not provided
11:40:42.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:42.720656 18966 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
11:40:42.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:42.721015 18966 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
11:40:42.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:42.725819 18966 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-1/data/instance:
11:40:42.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c716745b4a1b4b39957ab73274b4df06"
11:40:42.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 11:40:42 on dist-test-slave-h1fj"
11:40:42.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:42.726861 18966 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-1/wal/instance:
11:40:42.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c716745b4a1b4b39957ab73274b4df06"
11:40:42.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 11:40:42 on dist-test-slave-h1fj"
11:40:42.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:42.733726 18966 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.003s sys 0.003s
11:40:42.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:42.738761 18987 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
11:40:42.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:42.739720 18966 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
11:40:42.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:42.740049 18966 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-1/data,/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-1/wal
11:40:42.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c716745b4a1b4b39957ab73274b4df06"
11:40:42.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 11:40:42 on dist-test-slave-h1fj"
11:40:42.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:42.740345 18966 fs_report.cc:383] FS layout report
11:40:42.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
11:40:42.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-1/wal
11:40:42.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-1/wal
11:40:42.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-1/data/data
11:40:42.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
11:40:42.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
11:40:42.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
11:40:42.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
11:40:42.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
11:40:42.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
11:40:42.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
11:40:42.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
11:40:42.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
11:40:42.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
11:40:42.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
11:40:42.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:42.786092 18966 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
11:40:42.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:42.791518 18966 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
11:40:42.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:42.791950 18966 kserver.cc:163] Server-wide thread pool size limit: 3276
11:40:42.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:42.855780 18966 rpc_server.cc:307] RPC server started. Bound to: 127.18.108.125:36939
11:40:42.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:42.855858 19038 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.18.108.125:36939 every 8 connection(s)
11:40:42.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:42.858366 18966 server_base.cc:1166] Dumped server information to /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-1/data/info.pb
11:40:42.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:42.861344 18865 external_mini_cluster.cc:1388] Started /tmp/dist-test-task4Z5jUl/build/tsan/bin/kudu as pid 18966
11:40:42.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:42.861778 18865 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-1/wal/instance
11:40:42.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:42.863938 19039 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
11:40:42.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.18.108.124@KRBTEST.COM; defaulting to no policy
11:40:42.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:42.878001 19039 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.18.108.126" port: 43211 } has no permanent_uuid. Determining permanent_uuid...
11:40:42.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
11:40:42.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.18.108.124@KRBTEST.COM" created.
11:40:42.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 11:40:42 dist-test-slave-h1fj krb5kdc[18876](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790682042, etypes {rep=17 tkt=17 ses=17}, kudu/127.18.108.125@KRBTEST.COM for kudu/127.18.108.126@KRBTEST.COM
11:40:42.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
11:40:42.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.18.108.124 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/krb5kdc/kudu_127.18.108.124.keytab.
11:40:42.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.18.108.124 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/krb5kdc/kudu_127.18.108.124.keytab.
11:40:42.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:42.904285 18865 external_mini_cluster.cc:1326] Running /tmp/dist-test-task4Z5jUl/build/tsan/bin/kudu
11:40:42.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task4Z5jUl/build/tsan/bin/kudu
11:40:42.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-2/wal
11:40:42.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-2/data
11:40:42.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
11:40:42.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
11:40:42.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
11:40:42.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:40:42.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:40:42.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:40:42.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-2/logs
11:40:42.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-2/data/info.pb
11:40:42.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:40:42.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
11:40:42.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
11:40:42.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
11:40:42.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
11:40:42.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:40:42.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:40:42.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:40:42.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
11:40:42.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
11:40:42.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
11:40:42.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
11:40:42.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.108.124:42121
11:40:42.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.108.124
11:40:42.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
11:40:42.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/krb5kdc/kudu_127.18.108.124.keytab
11:40:42.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.18.108.124
11:40:42.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
11:40:42.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
11:40:42.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
11:40:42.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.108.84:34171
11:40:42.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:40:42.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:40:42.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:40:42.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
11:40:42.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.18.108.126:43211,127.18.108.125:36939,127.18.108.124:42121
11:40:42.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
11:40:42.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45823/jwks.json
11:40:42.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
11:40:42.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:42.911269 19039 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.18.108.125" port: 36939 } has no permanent_uuid. Determining permanent_uuid...
11:40:42.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 11:40:42 dist-test-slave-h1fj krb5kdc[18876](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790682042, etypes {rep=17 tkt=17 ses=17}, kudu/127.18.108.125@KRBTEST.COM for kudu/127.18.108.125@KRBTEST.COM
11:40:42.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:42.934461 19039 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 } has no permanent_uuid. Determining permanent_uuid...
11:40:42.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:42.936631 18990 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.18.108.124:42121: connect: Connection refused (error 111)
11:40:42.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:42.940088 19039 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.18.108.124:42121: Network error: Client connection negotiation failed: client connection to 127.18.108.124:42121: connect: Connection refused (error 111)
11:40:42.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:42.989467 19039 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 } attempt: 1
11:40:42.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:42.992861 19039 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.18.108.124:42121: Network error: Client connection negotiation failed: client connection to 127.18.108.124:42121: connect: Connection refused (error 111)
11:40:43.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:43.061192 19039 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 } attempt: 2
11:40:43.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:43.064603 19039 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.18.108.124:42121: Network error: Client connection negotiation failed: client connection to 127.18.108.124:42121: connect: Connection refused (error 111)
11:40:43.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:43.156023 19039 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 } attempt: 3
11:40:43.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:43.159399 19039 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.18.108.124:42121: Network error: Client connection negotiation failed: client connection to 127.18.108.124:42121: connect: Connection refused (error 111)
11:40:43.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:43.208249 19045 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
11:40:43.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:43.208832 19045 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
11:40:43.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:43.209476 19045 flags.cc:425] Enabled unsafe flag: --never_fsync=true
11:40:43.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:43.242841 19045 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
11:40:43.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:43.243044 19045 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
11:40:43.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:43.243350 19045 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
11:40:43.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:43.243551 19045 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
11:40:43.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:43.243734 19045 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
11:40:43.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:43.243920 19045 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
11:40:43.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:43.244055 19045 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:45823/jwks.json
11:40:43.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:43.279464 19045 master_runner.cc:386] Master server non-default flags:
11:40:43.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:40:43.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.108.84:34171
11:40:43.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:40:43.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:40:43.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-2/data
11:40:43.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-2/wal
11:40:43.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
11:40:43.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
11:40:43.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.18.108.126:43211,127.18.108.125:36939,127.18.108.124:42121
11:40:43.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/krb5kdc/kudu_127.18.108.124.keytab
11:40:43.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.18.108.124
11:40:43.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:40:43.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:40:43.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
11:40:43.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.108.124:42121
11:40:43.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
11:40:43.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
11:40:43.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
11:40:43.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45823/jwks.json
11:40:43.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
11:40:43.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
11:40:43.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
11:40:43.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:40:43.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:40:43.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-2/data/info.pb
11:40:43.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.108.124
11:40:43.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
11:40:43.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
11:40:43.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:40:43.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
11:40:43.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
11:40:43.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:40:43.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-2/logs
11:40:43.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:40:43.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
11:40:43.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
11:40:43.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
11:40:43.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
11:40:43.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
11:40:43.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
11:40:43.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 10:52:50 UTC on ccycloud.quasar-peqjre.root.comops.site
11:40:43.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679900
11:40:43.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
11:40:43.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:43.280586 19045 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
11:40:43.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:43.281512 19045 file_cache.cc:493] Constructed file cache file cache with capacity 419430
11:40:43.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:43.298635 19054 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:40:43.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:43.302796 19039 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 } attempt: 4
11:40:43.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:43.306492 19039 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.18.108.124:42121: Network error: Client connection negotiation failed: client connection to 127.18.108.124:42121: connect: Connection refused (error 111)
11:40:43.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:43.397625 18960 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.108.125" port: 36939 } attempt: 7
11:40:43.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 11:40:43 dist-test-slave-h1fj krb5kdc[18876](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790682040, etypes {rep=17 tkt=17 ses=17}, kudu/127.18.108.126@KRBTEST.COM for kudu/127.18.108.125@KRBTEST.COM
11:40:43.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:43.420186 18960 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 } has no permanent_uuid. Determining permanent_uuid...
11:40:43.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:43.425010 18960 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.18.108.124:42121: Network error: Client connection negotiation failed: client connection to 127.18.108.124:42121: connect: Connection refused (error 111)
11:40:43.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:43.483350 18960 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 } attempt: 1
11:40:43.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:43.486851 18960 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.18.108.124:42121: Network error: Client connection negotiation failed: client connection to 127.18.108.124:42121: connect: Connection refused (error 111)
11:40:43.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:43.568301 18960 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 } attempt: 2
11:40:43.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:43.571910 18960 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.18.108.124:42121: Network error: Client connection negotiation failed: client connection to 127.18.108.124:42121: connect: Connection refused (error 111)
11:40:43.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:43.605858 19039 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 } attempt: 5
11:40:43.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:43.609249 19039 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.18.108.124:42121: Network error: Client connection negotiation failed: client connection to 127.18.108.124:42121: connect: Connection refused (error 111)
11:40:43.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:43.657359 18960 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 } attempt: 3
11:40:43.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:43.661144 18960 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.18.108.124:42121: Network error: Client connection negotiation failed: client connection to 127.18.108.124:42121: connect: Connection refused (error 111)
11:40:43.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:43.801458 18960 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 } attempt: 4
11:40:43.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:43.805019 18960 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.18.108.124:42121: Network error: Client connection negotiation failed: client connection to 127.18.108.124:42121: connect: Connection refused (error 111)
11:40:44.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:44.088356 18960 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 } attempt: 5
11:40:44.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:44.091789 18960 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.18.108.124:42121: Network error: Client connection negotiation failed: client connection to 127.18.108.124:42121: connect: Connection refused (error 111)
11:40:44.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:44.156687 19039 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 } attempt: 6
11:40:44.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:44.162735 19039 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.18.108.124:42121: Network error: Client connection negotiation failed: client connection to 127.18.108.124:42121: connect: Connection refused (error 111)
11:40:44.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:43.299259 19056 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:40:44.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:43.299594 19053 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:40:44.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:44.414281 19055 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
11:40:44.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:44.414422 19045 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
11:40:44.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:44.417922 19045 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
11:40:44.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:44.420360 19045 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
11:40:44.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:44.421697 19045 hybrid_clock.cc:648] HybridClock initialized: now 1790682044421649 us; error 41 us; skew 500 ppm
11:40:44.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:44.422478 19045 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
11:40:44.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 11:40:44 dist-test-slave-h1fj krb5kdc[18876](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790682044, etypes {rep=17 tkt=17 ses=17}, kudu/127.18.108.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
11:40:44.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:44.427856 19045 init.cc:377] Logged in from keytab as kudu/127.18.108.124@KRBTEST.COM (short username kudu)
11:40:44.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:44.432652 19045 webserver.cc:460] Webserver started at http://127.18.108.124:37519/ using document root <none> and password file <none>
11:40:44.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:44.433534 19045 fs_manager.cc:359] Metadata directory not provided
11:40:44.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:44.433727 19045 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
11:40:44.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:44.434162 19045 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
11:40:44.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:44.439613 19045 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-2/data/instance:
11:40:44.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5a5e3302510a40e28f4aa8f781a651ea"
11:40:44.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 11:40:44 on dist-test-slave-h1fj"
11:40:44.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:44.440645 19045 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-2/wal/instance:
11:40:44.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5a5e3302510a40e28f4aa8f781a651ea"
11:40:44.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 11:40:44 on dist-test-slave-h1fj"
11:40:44.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:44.447497 19045 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.007s sys 0.000s
11:40:44.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:44.452569 19068 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
11:40:44.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:44.453660 19045 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.002s sys 0.003s
11:40:44.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:44.454090 19045 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-2/data,/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-2/wal
11:40:44.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5a5e3302510a40e28f4aa8f781a651ea"
11:40:44.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 11:40:44 on dist-test-slave-h1fj"
11:40:44.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:44.454391 19045 fs_report.cc:383] FS layout report
11:40:44.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
11:40:44.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-2/wal
11:40:44.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-2/wal
11:40:44.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-2/data/data
11:40:44.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
11:40:44.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
11:40:44.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
11:40:44.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
11:40:44.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
11:40:44.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
11:40:44.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
11:40:44.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
11:40:44.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
11:40:44.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
11:40:44.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
11:40:44.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:44.487434 19045 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
11:40:44.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:44.492360 19045 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
11:40:44.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:44.492782 19045 kserver.cc:163] Server-wide thread pool size limit: 3276
11:40:44.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:44.557294 19045 rpc_server.cc:307] RPC server started. Bound to: 127.18.108.124:42121
11:40:44.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:44.557385 19119 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.18.108.124:42121 every 8 connection(s)
11:40:44.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:44.559890 19045 server_base.cc:1166] Dumped server information to /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-2/data/info.pb
11:40:44.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:44.561331 18865 external_mini_cluster.cc:1388] Started /tmp/dist-test-task4Z5jUl/build/tsan/bin/kudu as pid 19045
11:40:44.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:44.561815 18865 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-2/wal/instance
11:40:44.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:44.565806 19120 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
11:40:44.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:44.577678 19120 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.18.108.126" port: 43211 } has no permanent_uuid. Determining permanent_uuid...
11:40:44.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 11:40:44 dist-test-slave-h1fj krb5kdc[18876](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790682039, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.18.108.124@KRBTEST.COM
11:40:44.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 11:40:44 dist-test-slave-h1fj krb5kdc[18876](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790682044, etypes {rep=17 tkt=17 ses=17}, kudu/127.18.108.124@KRBTEST.COM for kudu/127.18.108.126@KRBTEST.COM
11:40:44.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:44.609781 19120 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.18.108.125" port: 36939 } has no permanent_uuid. Determining permanent_uuid...
11:40:44.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 11:40:44 dist-test-slave-h1fj krb5kdc[18876](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790682044, etypes {rep=17 tkt=17 ses=17}, kudu/127.18.108.124@KRBTEST.COM for kudu/127.18.108.125@KRBTEST.COM
11:40:44.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 11:40:44 dist-test-slave-h1fj krb5kdc[18876](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790682039, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.18.108.125@KRBTEST.COM
11:40:44.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:44.632923 19120 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 } has no permanent_uuid. Determining permanent_uuid...
11:40:44.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 11:40:44 dist-test-slave-h1fj krb5kdc[18876](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790682044, etypes {rep=17 tkt=17 ses=17}, kudu/127.18.108.124@KRBTEST.COM for kudu/127.18.108.124@KRBTEST.COM
11:40:44.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:44.644114 18960 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 } attempt: 6
11:40:44.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 11:40:44 dist-test-slave-h1fj krb5kdc[18876](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790682039, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.18.108.126@KRBTEST.COM
11:40:44.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:44.669844 19120 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5a5e3302510a40e28f4aa8f781a651ea: Bootstrap starting.
11:40:44.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:44.676062 18865 external_mini_cluster.cc:930] 0 TS(s) registered with all masters
11:40:44.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:44.677352 19120 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 5a5e3302510a40e28f4aa8f781a651ea: Neither blocks nor log segments found. Creating new log.
11:40:44.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:44.681001 19120 log.cc:824] T 00000000000000000000000000000000 P 5a5e3302510a40e28f4aa8f781a651ea: Log is configured to *not* fsync() on all Append() calls
11:40:44.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 11:40:44 dist-test-slave-h1fj krb5kdc[18876](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790682040, etypes {rep=17 tkt=17 ses=17}, kudu/127.18.108.126@KRBTEST.COM for kudu/127.18.108.124@KRBTEST.COM
11:40:44.686 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
11:40:44.686 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
11:40:44.687 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
11:40:44.688 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
11:40:44.691 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.18.108.126"
port: 43211
}
bound_http_address {
host: "127.18.108.126"
port: 36579
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.18.108.125"
port: 36939
}
bound_http_address {
host: "127.18.108.125"
port: 38867
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.18.108.124"
port: 42121
}
bound_http_address {
host: "127.18.108.124"
port: 37519
}
}
}
11:40:44.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:44.691599 19120 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5a5e3302510a40e28f4aa8f781a651ea: No bootstrap required, opened a new log
11:40:44.692 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
11:40:44.692 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
11:40:44.692 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
11:40:44.693 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
11:40:44.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:44.706329 18960 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf: Bootstrap starting.
11:40:44.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:44.712867 18960 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf: Neither blocks nor log segments found. Creating new log.
11:40:44.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:44.715107 18960 log.cc:824] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf: Log is configured to *not* fsync() on all Append() calls
11:40:44.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:44.720050 19120 raft_consensus.cc:348] T 00000000000000000000000000000000 P 5a5e3302510a40e28f4aa8f781a651ea [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eb8f4e2d825e4e319fd704623a2750bf" member_type: VOTER last_known_addr { host: "127.18.108.126" port: 43211 } } peers { permanent_uuid: "c716745b4a1b4b39957ab73274b4df06" member_type: VOTER last_known_addr { host: "127.18.108.125" port: 36939 } } peers { permanent_uuid: "5a5e3302510a40e28f4aa8f781a651ea" member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 } }
11:40:44.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:44.720597 19120 raft_consensus.cc:374] T 00000000000000000000000000000000 P 5a5e3302510a40e28f4aa8f781a651ea [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
11:40:44.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:44.720793 19120 raft_consensus.cc:729] T 00000000000000000000000000000000 P 5a5e3302510a40e28f4aa8f781a651ea [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5a5e3302510a40e28f4aa8f781a651ea, State: Initialized, Role: FOLLOWER
11:40:44.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 11:40:44 dist-test-slave-h1fj krb5kdc[18876](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790682044, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
11:40:44.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
11:40:44.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:44.721840 18960 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf: No bootstrap required, opened a new log
11:40:44.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:44.721520 19120 consensus_queue.cc:260] T 00000000000000000000000000000000 P 5a5e3302510a40e28f4aa8f781a651ea [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eb8f4e2d825e4e319fd704623a2750bf" member_type: VOTER last_known_addr { host: "127.18.108.126" port: 43211 } } peers { permanent_uuid: "c716745b4a1b4b39957ab73274b4df06" member_type: VOTER last_known_addr { host: "127.18.108.125" port: 36939 } } peers { permanent_uuid: "5a5e3302510a40e28f4aa8f781a651ea" member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 } }
11:40:44.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:44.724300 19135 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5a5e3302510a40e28f4aa8f781a651ea [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eb8f4e2d825e4e319fd704623a2750bf" member_type: VOTER last_known_addr { host: "127.18.108.126" port: 43211 } } peers { permanent_uuid: "c716745b4a1b4b39957ab73274b4df06" member_type: VOTER last_known_addr { host: "127.18.108.125" port: 36939 } } peers { permanent_uuid: "5a5e3302510a40e28f4aa8f781a651ea" member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 } } }
11:40:44.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:44.725261 19135 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5a5e3302510a40e28f4aa8f781a651ea [sys.catalog]: This master's current role is: FOLLOWER
11:40:44.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:44.726156 19120 sys_catalog.cc:564] T 00000000000000000000000000000000 P 5a5e3302510a40e28f4aa8f781a651ea [sys.catalog]: configured and running, proceeding with master startup.
11:40:44.729 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
11:40:44.730 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/krb5kdc/krb5cc
11:40:44.732 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
11:40:44.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:44.747720 19074 tablet.cc:2369] T 00000000000000000000000000000000 P 5a5e3302510a40e28f4aa8f781a651ea: Can't schedule compaction. Clean time has not been advanced past its initial value.
11:40:44.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:44.747807 18960 raft_consensus.cc:348] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eb8f4e2d825e4e319fd704623a2750bf" member_type: VOTER last_known_addr { host: "127.18.108.126" port: 43211 } } peers { permanent_uuid: "c716745b4a1b4b39957ab73274b4df06" member_type: VOTER last_known_addr { host: "127.18.108.125" port: 36939 } } peers { permanent_uuid: "5a5e3302510a40e28f4aa8f781a651ea" member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 } }
11:40:44.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:44.748569 18960 raft_consensus.cc:374] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
11:40:44.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:44.748867 18960 raft_consensus.cc:729] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: eb8f4e2d825e4e319fd704623a2750bf, State: Initialized, Role: FOLLOWER
11:40:44.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:44.750283 18960 consensus_queue.cc:260] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eb8f4e2d825e4e319fd704623a2750bf" member_type: VOTER last_known_addr { host: "127.18.108.126" port: 43211 } } peers { permanent_uuid: "c716745b4a1b4b39957ab73274b4df06" member_type: VOTER last_known_addr { host: "127.18.108.125" port: 36939 } } peers { permanent_uuid: "5a5e3302510a40e28f4aa8f781a651ea" member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 } }
11:40:44.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:44.753274 19146 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 5a5e3302510a40e28f4aa8f781a651ea: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:40:44.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:44.753736 19146 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:40:44.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:44.754909 19148 sys_catalog.cc:455] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eb8f4e2d825e4e319fd704623a2750bf" member_type: VOTER last_known_addr { host: "127.18.108.126" port: 43211 } } peers { permanent_uuid: "c716745b4a1b4b39957ab73274b4df06" member_type: VOTER last_known_addr { host: "127.18.108.125" port: 36939 } } peers { permanent_uuid: "5a5e3302510a40e28f4aa8f781a651ea" member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 } } }
11:40:44.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:44.755856 19148 sys_catalog.cc:458] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [sys.catalog]: This master's current role is: FOLLOWER
11:40:44.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:44.756598 18960 sys_catalog.cc:564] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [sys.catalog]: configured and running, proceeding with master startup.
11:40:44.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:44.759598 19148 raft_consensus.cc:482] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
11:40:44.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:44.760043 19148 raft_consensus.cc:504] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eb8f4e2d825e4e319fd704623a2750bf" member_type: VOTER last_known_addr { host: "127.18.108.126" port: 43211 } } peers { permanent_uuid: "c716745b4a1b4b39957ab73274b4df06" member_type: VOTER last_known_addr { host: "127.18.108.125" port: 36939 } } peers { permanent_uuid: "5a5e3302510a40e28f4aa8f781a651ea" member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 } }
11:40:44.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:44.762240 19148 leader_election.cc:288] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers c716745b4a1b4b39957ab73274b4df06 (127.18.108.125:36939), 5a5e3302510a40e28f4aa8f781a651ea (127.18.108.124:42121)
11:40:44.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:44.765156 19095 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "eb8f4e2d825e4e319fd704623a2750bf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5a5e3302510a40e28f4aa8f781a651ea" is_pre_election: true
11:40:44.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:44.766060 19095 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 5a5e3302510a40e28f4aa8f781a651ea [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate eb8f4e2d825e4e319fd704623a2750bf in term 0.
11:40:44.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:44.767483 18911 leader_election.cc:302] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 5a5e3302510a40e28f4aa8f781a651ea, eb8f4e2d825e4e319fd704623a2750bf; no voters:
11:40:44.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:44.768450 19148 raft_consensus.cc:2781] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [term 0 FOLLOWER]: Leader pre-election won for term 1
11:40:44.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:44.769048 19148 raft_consensus.cc:482] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
11:40:44.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:44.769840 19148 raft_consensus.cc:3037] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [term 0 FOLLOWER]: Advancing to term 1
11:40:44.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:44.781143 19148 raft_consensus.cc:504] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eb8f4e2d825e4e319fd704623a2750bf" member_type: VOTER last_known_addr { host: "127.18.108.126" port: 43211 } } peers { permanent_uuid: "c716745b4a1b4b39957ab73274b4df06" member_type: VOTER last_known_addr { host: "127.18.108.125" port: 36939 } } peers { permanent_uuid: "5a5e3302510a40e28f4aa8f781a651ea" member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 } }
11:40:44.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:44.750200 19045 master_runner.cc:186] Error getting master registration for 127.18.108.126:43211: OK, instance_id { permanent_uuid: "eb8f4e2d825e4e319fd704623a2750bf" instance_seqno: 1790682041008597 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
11:40:44.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:44.785840 19045 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
11:40:44.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:44.786249 19148 leader_election.cc:288] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [CANDIDATE]: Term 1 election: Requested vote from peers c716745b4a1b4b39957ab73274b4df06 (127.18.108.125:36939), 5a5e3302510a40e28f4aa8f781a651ea (127.18.108.124:42121)
11:40:44.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:44.787142 19095 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "eb8f4e2d825e4e319fd704623a2750bf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5a5e3302510a40e28f4aa8f781a651ea"
11:40:44.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:44.787714 19095 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 5a5e3302510a40e28f4aa8f781a651ea [term 0 FOLLOWER]: Advancing to term 1
11:40:44.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:44.793872 19095 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 5a5e3302510a40e28f4aa8f781a651ea [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate eb8f4e2d825e4e319fd704623a2750bf in term 1.
11:40:44.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:44.794983 18911 leader_election.cc:302] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 5a5e3302510a40e28f4aa8f781a651ea, eb8f4e2d825e4e319fd704623a2750bf; no voters:
11:40:44.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:44.795709 19148 raft_consensus.cc:2781] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [term 1 FOLLOWER]: Leader election won for term 1
11:40:44.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:44.798734 19148 raft_consensus.cc:686] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [term 1 LEADER]: Becoming Leader. State: Replica: eb8f4e2d825e4e319fd704623a2750bf, State: Running, Role: LEADER
11:40:44.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:44.799721 19148 consensus_queue.cc:237] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eb8f4e2d825e4e319fd704623a2750bf" member_type: VOTER last_known_addr { host: "127.18.108.126" port: 43211 } } peers { permanent_uuid: "c716745b4a1b4b39957ab73274b4df06" member_type: VOTER last_known_addr { host: "127.18.108.125" port: 36939 } } peers { permanent_uuid: "5a5e3302510a40e28f4aa8f781a651ea" member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 } }
11:40:44.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:44.814597 19156 sys_catalog.cc:455] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [sys.catalog]: SysCatalogTable state changed. Reason: New leader eb8f4e2d825e4e319fd704623a2750bf. Latest consensus state: current_term: 1 leader_uuid: "eb8f4e2d825e4e319fd704623a2750bf" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eb8f4e2d825e4e319fd704623a2750bf" member_type: VOTER last_known_addr { host: "127.18.108.126" port: 43211 } } peers { permanent_uuid: "c716745b4a1b4b39957ab73274b4df06" member_type: VOTER last_known_addr { host: "127.18.108.125" port: 36939 } } peers { permanent_uuid: "5a5e3302510a40e28f4aa8f781a651ea" member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 } } }
11:40:44.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:44.816121 19156 sys_catalog.cc:458] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [sys.catalog]: This master's current role is: LEADER
11:40:44.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:44.841766 19161 catalog_manager.cc:1558] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:40:44.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:44.842412 19161 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:40:44.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:44.843196 19162 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
11:40:44.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:44.821200 18891 master_runner.cc:186] Error getting master registration for 127.18.108.125:36939: OK, instance_id { permanent_uuid: "c716745b4a1b4b39957ab73274b4df06" instance_seqno: 1790682042838287 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
11:40:44.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:44.854297 18891 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
11:40:44.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:44.858028 19162 catalog_manager.cc:1484] Initializing Kudu cluster ID...
11:40:44.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:44.894140 19095 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 5a5e3302510a40e28f4aa8f781a651ea [term 1 FOLLOWER]: Refusing update from remote peer eb8f4e2d825e4e319fd704623a2750bf: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
11:40:44.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:44.896673 19156 consensus_queue.cc:1035] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [LEADER]: Connected to new peer: Peer: permanent_uuid: "5a5e3302510a40e28f4aa8f781a651ea" member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
11:40:44.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:44.922183 19164 mvcc.cc:205] Tried to move back new op lower bound from 7334633655870246912 to 7334633655508836352. Current Snapshot: MvccSnapshot[applied={T|T < 7334633655870246912}]
11:40:44.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:44.924649 19135 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5a5e3302510a40e28f4aa8f781a651ea [sys.catalog]: SysCatalogTable state changed. Reason: New leader eb8f4e2d825e4e319fd704623a2750bf. Latest consensus state: current_term: 1 leader_uuid: "eb8f4e2d825e4e319fd704623a2750bf" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eb8f4e2d825e4e319fd704623a2750bf" member_type: VOTER last_known_addr { host: "127.18.108.126" port: 43211 } } peers { permanent_uuid: "c716745b4a1b4b39957ab73274b4df06" member_type: VOTER last_known_addr { host: "127.18.108.125" port: 36939 } } peers { permanent_uuid: "5a5e3302510a40e28f4aa8f781a651ea" member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 } } }
11:40:44.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:44.925420 19135 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5a5e3302510a40e28f4aa8f781a651ea [sys.catalog]: This master's current role is: FOLLOWER
11:40:44.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:44.927567 19156 sys_catalog.cc:455] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "eb8f4e2d825e4e319fd704623a2750bf" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eb8f4e2d825e4e319fd704623a2750bf" member_type: VOTER last_known_addr { host: "127.18.108.126" port: 43211 } } peers { permanent_uuid: "c716745b4a1b4b39957ab73274b4df06" member_type: VOTER last_known_addr { host: "127.18.108.125" port: 36939 } } peers { permanent_uuid: "5a5e3302510a40e28f4aa8f781a651ea" member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 } } }
11:40:44.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:44.929113 19156 sys_catalog.cc:458] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [sys.catalog]: This master's current role is: LEADER
11:40:44.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:44.930558 19162 catalog_manager.cc:1347] Generated new cluster ID: 09fd2865e33b46ce9a354093f255891e
11:40:44.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:44.930805 19162 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
11:40:44.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:44.934852 19135 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5a5e3302510a40e28f4aa8f781a651ea [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "eb8f4e2d825e4e319fd704623a2750bf" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eb8f4e2d825e4e319fd704623a2750bf" member_type: VOTER last_known_addr { host: "127.18.108.126" port: 43211 } } peers { permanent_uuid: "c716745b4a1b4b39957ab73274b4df06" member_type: VOTER last_known_addr { host: "127.18.108.125" port: 36939 } } peers { permanent_uuid: "5a5e3302510a40e28f4aa8f781a651ea" member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 } } }
11:40:44.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:44.935573 19135 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5a5e3302510a40e28f4aa8f781a651ea [sys.catalog]: This master's current role is: FOLLOWER
11:40:44.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:44.954372 19162 catalog_manager.cc:1370] Generated new certificate authority record
11:40:44.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:44.957885 19162 catalog_manager.cc:1504] Loading token signing keys...
11:40:44.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:44.985857 19162 catalog_manager.cc:5868] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf: Generated new TSK 0
11:40:44.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:44.987072 19162 catalog_manager.cc:1514] Initializing in-progress tserver states...
11:40:45.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:45.223170 19039 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 } attempt: 7
11:40:45.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 11:40:45 dist-test-slave-h1fj krb5kdc[18876](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790682042, etypes {rep=17 tkt=17 ses=17}, kudu/127.18.108.125@KRBTEST.COM for kudu/127.18.108.124@KRBTEST.COM
11:40:45.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:45.262148 19039 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06: Bootstrap starting.
11:40:45.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:45.269527 19039 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06: Neither blocks nor log segments found. Creating new log.
11:40:45.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:45.271921 19039 log.cc:824] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06: Log is configured to *not* fsync() on all Append() calls
11:40:45.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:45.277693 19039 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06: No bootstrap required, opened a new log
11:40:45.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:45.308751 19039 raft_consensus.cc:348] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eb8f4e2d825e4e319fd704623a2750bf" member_type: VOTER last_known_addr { host: "127.18.108.126" port: 43211 } } peers { permanent_uuid: "c716745b4a1b4b39957ab73274b4df06" member_type: VOTER last_known_addr { host: "127.18.108.125" port: 36939 } } peers { permanent_uuid: "5a5e3302510a40e28f4aa8f781a651ea" member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 } }
11:40:45.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:45.309551 19039 raft_consensus.cc:374] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
11:40:45.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:45.309815 19039 raft_consensus.cc:729] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c716745b4a1b4b39957ab73274b4df06, State: Initialized, Role: FOLLOWER
11:40:45.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:45.310859 19039 consensus_queue.cc:260] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eb8f4e2d825e4e319fd704623a2750bf" member_type: VOTER last_known_addr { host: "127.18.108.126" port: 43211 } } peers { permanent_uuid: "c716745b4a1b4b39957ab73274b4df06" member_type: VOTER last_known_addr { host: "127.18.108.125" port: 36939 } } peers { permanent_uuid: "5a5e3302510a40e28f4aa8f781a651ea" member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 } }
11:40:45.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:45.315467 19039 sys_catalog.cc:564] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [sys.catalog]: configured and running, proceeding with master startup.
11:40:45.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:45.315662 19014 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "eb8f4e2d825e4e319fd704623a2750bf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c716745b4a1b4b39957ab73274b4df06" is_pre_election: true
11:40:45.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:45.314492 19173 sys_catalog.cc:455] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eb8f4e2d825e4e319fd704623a2750bf" member_type: VOTER last_known_addr { host: "127.18.108.126" port: 43211 } } peers { permanent_uuid: "c716745b4a1b4b39957ab73274b4df06" member_type: VOTER last_known_addr { host: "127.18.108.125" port: 36939 } } peers { permanent_uuid: "5a5e3302510a40e28f4aa8f781a651ea" member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 } } }
11:40:45.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:45.317276 19173 sys_catalog.cc:458] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [sys.catalog]: This master's current role is: FOLLOWER
11:40:45.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:45.316459 19013 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "eb8f4e2d825e4e319fd704623a2750bf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c716745b4a1b4b39957ab73274b4df06"
11:40:45.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:45.318627 19012 raft_consensus.cc:3037] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [term 0 FOLLOWER]: Advancing to term 1
11:40:45.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:45.326088 19012 raft_consensus.cc:1261] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [term 1 FOLLOWER]: Refusing update from remote peer eb8f4e2d825e4e319fd704623a2750bf: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
11:40:45.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:45.327670 19156 consensus_queue.cc:1035] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [LEADER]: Connected to new peer: Peer: permanent_uuid: "c716745b4a1b4b39957ab73274b4df06" member_type: VOTER last_known_addr { host: "127.18.108.125" port: 36939 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
11:40:45.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:45.385545 19173 sys_catalog.cc:455] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [sys.catalog]: SysCatalogTable state changed. Reason: New leader eb8f4e2d825e4e319fd704623a2750bf. Latest consensus state: current_term: 1 leader_uuid: "eb8f4e2d825e4e319fd704623a2750bf" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eb8f4e2d825e4e319fd704623a2750bf" member_type: VOTER last_known_addr { host: "127.18.108.126" port: 43211 } } peers { permanent_uuid: "c716745b4a1b4b39957ab73274b4df06" member_type: VOTER last_known_addr { host: "127.18.108.125" port: 36939 } } peers { permanent_uuid: "5a5e3302510a40e28f4aa8f781a651ea" member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 } } }
11:40:45.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:45.387387 19173 sys_catalog.cc:458] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [sys.catalog]: This master's current role is: FOLLOWER
11:40:45.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:45.387192 19179 sys_catalog.cc:455] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "eb8f4e2d825e4e319fd704623a2750bf" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eb8f4e2d825e4e319fd704623a2750bf" member_type: VOTER last_known_addr { host: "127.18.108.126" port: 43211 } } peers { permanent_uuid: "c716745b4a1b4b39957ab73274b4df06" member_type: VOTER last_known_addr { host: "127.18.108.125" port: 36939 } } peers { permanent_uuid: "5a5e3302510a40e28f4aa8f781a651ea" member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 } } }
11:40:45.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:45.388839 19179 sys_catalog.cc:458] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [sys.catalog]: This master's current role is: FOLLOWER
11:40:45.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:45.416795 19189 catalog_manager.cc:1259] Loaded cluster ID: 09fd2865e33b46ce9a354093f255891e
11:40:45.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:45.417165 19189 catalog_manager.cc:1552] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06: loading cluster ID for follower catalog manager: success
11:40:45.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:45.424616 19189 catalog_manager.cc:1574] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06: acquiring CA information for follower catalog manager: success
11:40:45.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:45.430428 19189 catalog_manager.cc:1602] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
11:40:45.732 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.18.108.126:43211
11:40:45.734 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
11:40:45.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:45.734964 18865 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task4Z5jUl/build/tsan/bin/kudu with pid 18891
11:40:45.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:45.759138 19146 catalog_manager.cc:1259] Loaded cluster ID: 09fd2865e33b46ce9a354093f255891e
11:40:45.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:45.759654 19146 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 5a5e3302510a40e28f4aa8f781a651ea: loading cluster ID for follower catalog manager: success
11:40:45.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:45.767169 19146 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 5a5e3302510a40e28f4aa8f781a651ea: acquiring CA information for follower catalog manager: success
11:40:45.768 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
11:40:45.768 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.18.108.125:36939
11:40:45.768 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
11:40:45.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:45.768767 18865 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task4Z5jUl/build/tsan/bin/kudu with pid 18966
11:40:45.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:45.779862 19146 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 5a5e3302510a40e28f4aa8f781a651ea: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
11:40:45.792 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
11:40:45.792 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.18.108.124:42121
11:40:45.792 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
11:40:45.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:45.792670 18865 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task4Z5jUl/build/tsan/bin/kudu with pid 19045
11:40:45.812 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
11:40:45.812 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.18.108.126:43211
11:40:45.813 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
11:40:45.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:45.815105 18865 external_mini_cluster.cc:1326] Running /tmp/dist-test-task4Z5jUl/build/tsan/bin/kudu
11:40:45.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task4Z5jUl/build/tsan/bin/kudu
11:40:45.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-0/wal
11:40:45.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-0/data
11:40:45.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
11:40:45.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
11:40:45.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
11:40:45.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:40:45.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:40:45.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:40:45.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-0/logs
11:40:45.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-0/data/info.pb
11:40:45.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:40:45.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
11:40:45.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
11:40:45.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
11:40:45.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
11:40:45.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:40:45.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:40:45.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:40:45.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
11:40:45.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
11:40:45.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
11:40:45.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
11:40:45.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.108.126:43211
11:40:45.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.108.126
11:40:45.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36579
11:40:45.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/krb5kdc/kudu_127.18.108.126.keytab
11:40:45.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.18.108.126
11:40:45.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
11:40:45.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
11:40:45.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
11:40:45.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.108.84:34171
11:40:45.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:40:45.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:40:45.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:40:45.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
11:40:45.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.18.108.126:43211,127.18.108.125:36939,127.18.108.124:42121
11:40:45.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
11:40:45.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45823/jwks.json
11:40:45.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
11:40:46.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:46.107360 19192 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
11:40:46.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:46.107947 19192 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
11:40:46.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:46.108664 19192 flags.cc:425] Enabled unsafe flag: --never_fsync=true
11:40:46.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:46.139590 19192 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
11:40:46.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:46.139878 19192 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
11:40:46.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:46.140254 19192 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
11:40:46.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:46.140537 19192 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
11:40:46.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:46.140810 19192 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
11:40:46.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:46.141079 19192 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
11:40:46.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:46.141281 19192 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:45823/jwks.json
11:40:46.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:46.178812 19192 master_runner.cc:386] Master server non-default flags:
11:40:46.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:40:46.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.108.84:34171
11:40:46.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:40:46.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:40:46.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-0/data
11:40:46.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-0/wal
11:40:46.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
11:40:46.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
11:40:46.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.18.108.126:43211,127.18.108.125:36939,127.18.108.124:42121
11:40:46.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/krb5kdc/kudu_127.18.108.126.keytab
11:40:46.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.18.108.126
11:40:46.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:40:46.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:40:46.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
11:40:46.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.108.126:43211
11:40:46.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
11:40:46.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
11:40:46.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
11:40:46.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45823/jwks.json
11:40:46.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
11:40:46.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
11:40:46.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
11:40:46.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:40:46.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:40:46.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-0/data/info.pb
11:40:46.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.108.126
11:40:46.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36579
11:40:46.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
11:40:46.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:40:46.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
11:40:46.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
11:40:46.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:40:46.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-0/logs
11:40:46.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:40:46.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
11:40:46.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
11:40:46.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
11:40:46.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
11:40:46.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
11:40:46.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
11:40:46.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 10:52:50 UTC on ccycloud.quasar-peqjre.root.comops.site
11:40:46.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679900
11:40:46.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
11:40:46.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:46.180013 19192 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
11:40:46.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:46.180924 19192 file_cache.cc:493] Constructed file cache file cache with capacity 419430
11:40:46.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:46.198448 19199 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:40:47.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:46.198539 19200 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:40:47.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:46.199550 19192 server_base.cc:1034] running on GCE node
11:40:47.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:46.198549 19202 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:40:47.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:47.303921 19192 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
11:40:47.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:47.306766 19192 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
11:40:47.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:47.308156 19192 hybrid_clock.cc:648] HybridClock initialized: now 1790682047308122 us; error 43 us; skew 500 ppm
11:40:47.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:47.308959 19192 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
11:40:47.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 11:40:47 dist-test-slave-h1fj krb5kdc[18876](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790682047, etypes {rep=17 tkt=17 ses=17}, kudu/127.18.108.126@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
11:40:47.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:47.318367 19192 init.cc:377] Logged in from keytab as kudu/127.18.108.126@KRBTEST.COM (short username kudu)
11:40:47.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:47.323417 19192 webserver.cc:460] Webserver started at http://127.18.108.126:36579/ using document root <none> and password file <none>
11:40:47.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:47.324225 19192 fs_manager.cc:359] Metadata directory not provided
11:40:47.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:47.324409 19192 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
11:40:47.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:47.331905 19192 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.006s sys 0.001s
11:40:47.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:47.336051 19211 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
11:40:47.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:47.337080 19192 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.004s sys 0.001s
11:40:47.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:47.337363 19192 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-0/data,/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-0/wal
11:40:47.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "eb8f4e2d825e4e319fd704623a2750bf"
11:40:47.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 11:40:40 on dist-test-slave-h1fj"
11:40:47.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:47.339570 19192 fs_report.cc:383] FS layout report
11:40:47.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
11:40:47.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-0/wal
11:40:47.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-0/wal
11:40:47.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-0/data/data
11:40:47.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
11:40:47.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
11:40:47.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
11:40:47.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
11:40:47.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
11:40:47.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
11:40:47.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
11:40:47.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
11:40:47.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
11:40:47.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
11:40:47.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
11:40:47.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:47.368541 19192 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
11:40:47.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:47.373786 19192 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
11:40:47.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:47.374166 19192 kserver.cc:163] Server-wide thread pool size limit: 3276
11:40:47.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:47.439682 19192 rpc_server.cc:307] RPC server started. Bound to: 127.18.108.126:43211
11:40:47.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:47.439862 19262 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.18.108.126:43211 every 8 connection(s)
11:40:47.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:47.442203 19192 server_base.cc:1166] Dumped server information to /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-0/data/info.pb
11:40:47.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:47.446585 18865 external_mini_cluster.cc:1388] Started /tmp/dist-test-task4Z5jUl/build/tsan/bin/kudu as pid 19192
11:40:47.447 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
11:40:47.447 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.18.108.125:36939
11:40:47.447 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
11:40:47.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:47.449226 18865 external_mini_cluster.cc:1326] Running /tmp/dist-test-task4Z5jUl/build/tsan/bin/kudu
11:40:47.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task4Z5jUl/build/tsan/bin/kudu
11:40:47.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-1/wal
11:40:47.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-1/data
11:40:47.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
11:40:47.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
11:40:47.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
11:40:47.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:40:47.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:40:47.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:40:47.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-1/logs
11:40:47.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-1/data/info.pb
11:40:47.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:40:47.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
11:40:47.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
11:40:47.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
11:40:47.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
11:40:47.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:40:47.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:40:47.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:40:47.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
11:40:47.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
11:40:47.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
11:40:47.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
11:40:47.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.108.125:36939
11:40:47.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.108.125
11:40:47.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38867
11:40:47.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/krb5kdc/kudu_127.18.108.125.keytab
11:40:47.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.18.108.125
11:40:47.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
11:40:47.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
11:40:47.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
11:40:47.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.108.84:34171
11:40:47.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:40:47.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:40:47.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:40:47.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
11:40:47.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.18.108.126:43211,127.18.108.125:36939,127.18.108.124:42121
11:40:47.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
11:40:47.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45823/jwks.json
11:40:47.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
11:40:47.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:47.453549 19263 sys_catalog.cc:263] Verifying existing consensus state
11:40:47.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:47.459257 19263 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf: Bootstrap starting.
11:40:47.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:47.493476 19263 log.cc:824] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf: Log is configured to *not* fsync() on all Append() calls
11:40:47.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:47.507566 19263 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
11:40:47.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:47.508394 19263 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf: Bootstrap complete.
11:40:47.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:47.531478 19263 raft_consensus.cc:348] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eb8f4e2d825e4e319fd704623a2750bf" member_type: VOTER last_known_addr { host: "127.18.108.126" port: 43211 } } peers { permanent_uuid: "c716745b4a1b4b39957ab73274b4df06" member_type: VOTER last_known_addr { host: "127.18.108.125" port: 36939 } } peers { permanent_uuid: "5a5e3302510a40e28f4aa8f781a651ea" member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 } }
11:40:47.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:47.533969 19263 raft_consensus.cc:729] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: eb8f4e2d825e4e319fd704623a2750bf, State: Initialized, Role: FOLLOWER
11:40:47.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:47.534801 19263 consensus_queue.cc:260] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eb8f4e2d825e4e319fd704623a2750bf" member_type: VOTER last_known_addr { host: "127.18.108.126" port: 43211 } } peers { permanent_uuid: "c716745b4a1b4b39957ab73274b4df06" member_type: VOTER last_known_addr { host: "127.18.108.125" port: 36939 } } peers { permanent_uuid: "5a5e3302510a40e28f4aa8f781a651ea" member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 } }
11:40:47.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:47.537492 19267 sys_catalog.cc:455] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eb8f4e2d825e4e319fd704623a2750bf" member_type: VOTER last_known_addr { host: "127.18.108.126" port: 43211 } } peers { permanent_uuid: "c716745b4a1b4b39957ab73274b4df06" member_type: VOTER last_known_addr { host: "127.18.108.125" port: 36939 } } peers { permanent_uuid: "5a5e3302510a40e28f4aa8f781a651ea" member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 } } }
11:40:47.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:47.538298 19267 sys_catalog.cc:458] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [sys.catalog]: This master's current role is: FOLLOWER
11:40:47.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:47.539075 19263 sys_catalog.cc:564] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [sys.catalog]: configured and running, proceeding with master startup.
11:40:47.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:47.571973 19278 catalog_manager.cc:1259] Loaded cluster ID: 09fd2865e33b46ce9a354093f255891e
11:40:47.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:47.573097 19278 catalog_manager.cc:1552] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf: loading cluster ID for follower catalog manager: success
11:40:47.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:47.581427 19278 catalog_manager.cc:1574] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf: acquiring CA information for follower catalog manager: success
11:40:47.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:47.587142 19278 catalog_manager.cc:1602] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
11:40:47.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 11:40:47 dist-test-slave-h1fj krb5kdc[18876](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790682047, etypes {rep=17 tkt=17 ses=17}, kudu/127.18.108.126@KRBTEST.COM for kudu/127.18.108.126@KRBTEST.COM
11:40:47.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:47.611312 19192 master_runner.cc:186] Error getting master registration for 127.18.108.125:36939: Network error: Client connection negotiation failed: client connection to 127.18.108.125:36939: connect: Connection refused (error 111),
11:40:47.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:47.640568 19192 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
11:40:47.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:47.769440 19265 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
11:40:47.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:47.770002 19265 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
11:40:47.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:47.770725 19265 flags.cc:425] Enabled unsafe flag: --never_fsync=true
11:40:47.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:47.801186 19265 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
11:40:47.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:47.801384 19265 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
11:40:47.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:47.801632 19265 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
11:40:47.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:47.801841 19265 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
11:40:47.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:47.802027 19265 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
11:40:47.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:47.802265 19265 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
11:40:47.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:47.802423 19265 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:45823/jwks.json
11:40:47.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:47.838023 19265 master_runner.cc:386] Master server non-default flags:
11:40:47.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:40:47.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.108.84:34171
11:40:47.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:40:47.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:40:47.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-1/data
11:40:47.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-1/wal
11:40:47.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
11:40:47.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
11:40:47.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.18.108.126:43211,127.18.108.125:36939,127.18.108.124:42121
11:40:47.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/krb5kdc/kudu_127.18.108.125.keytab
11:40:47.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.18.108.125
11:40:47.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:40:47.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:40:47.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
11:40:47.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.108.125:36939
11:40:47.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
11:40:47.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
11:40:47.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
11:40:47.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45823/jwks.json
11:40:47.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
11:40:47.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
11:40:47.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
11:40:47.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:40:47.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:40:47.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-1/data/info.pb
11:40:47.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.108.125
11:40:47.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38867
11:40:47.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
11:40:47.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:40:47.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
11:40:47.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
11:40:47.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:40:47.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-1/logs
11:40:47.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:40:47.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
11:40:47.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
11:40:47.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
11:40:47.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
11:40:47.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
11:40:47.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
11:40:47.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 10:52:50 UTC on ccycloud.quasar-peqjre.root.comops.site
11:40:47.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679900
11:40:47.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
11:40:47.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:47.839114 19265 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
11:40:47.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:47.840137 19265 file_cache.cc:493] Constructed file cache file cache with capacity 419430
11:40:47.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:47.858032 19289 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:40:49.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:49.255406 19295 raft_consensus.cc:482] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
11:40:49.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:49.256438 19295 raft_consensus.cc:504] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eb8f4e2d825e4e319fd704623a2750bf" member_type: VOTER last_known_addr { host: "127.18.108.126" port: 43211 } } peers { permanent_uuid: "c716745b4a1b4b39957ab73274b4df06" member_type: VOTER last_known_addr { host: "127.18.108.125" port: 36939 } } peers { permanent_uuid: "5a5e3302510a40e28f4aa8f781a651ea" member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 } }
11:40:49.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:47.858817 19288 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:40:49.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:47.860013 19291 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:40:49.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:49.268791 19212 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.18.108.125:36939: connect: Connection refused (error 111)
11:40:49.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:49.267705 19265 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
11:40:49.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:49.267828 19290 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1410 milliseconds
11:40:49.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:49.270480 19295 leader_election.cc:288] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers c716745b4a1b4b39957ab73274b4df06 (127.18.108.125:36939), 5a5e3302510a40e28f4aa8f781a651ea (127.18.108.124:42121)
11:40:49.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:49.274433 19265 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
11:40:49.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:49.276894 19212 leader_election.cc:334] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer c716745b4a1b4b39957ab73274b4df06 (127.18.108.125:36939): Network error: Client connection negotiation failed: client connection to 127.18.108.125:36939: connect: Connection refused (error 111)
11:40:49.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:49.278232 19214 leader_election.cc:334] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 5a5e3302510a40e28f4aa8f781a651ea (127.18.108.124:42121): Network error: Client connection negotiation failed: client connection to 127.18.108.124:42121: connect: Connection refused (error 111)
11:40:49.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:49.278517 19265 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
11:40:49.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:49.278622 19214 leader_election.cc:302] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: eb8f4e2d825e4e319fd704623a2750bf; no voters: 5a5e3302510a40e28f4aa8f781a651ea, c716745b4a1b4b39957ab73274b4df06
11:40:49.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:49.279716 19295 raft_consensus.cc:2726] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
11:40:49.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:49.279919 19265 hybrid_clock.cc:648] HybridClock initialized: now 1790682049279862 us; error 47 us; skew 500 ppm
11:40:49.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:49.281010 19265 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
11:40:49.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 11:40:49 dist-test-slave-h1fj krb5kdc[18876](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790682049, etypes {rep=17 tkt=17 ses=17}, kudu/127.18.108.125@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
11:40:49.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:49.287807 19265 init.cc:377] Logged in from keytab as kudu/127.18.108.125@KRBTEST.COM (short username kudu)
11:40:49.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:49.293188 19265 webserver.cc:460] Webserver started at http://127.18.108.125:38867/ using document root <none> and password file <none>
11:40:49.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:49.294014 19265 fs_manager.cc:359] Metadata directory not provided
11:40:49.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:49.294288 19265 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
11:40:49.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:49.302582 19265 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.006s sys 0.000s
11:40:49.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:49.307500 19304 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
11:40:49.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:49.308667 19265 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.001s sys 0.003s
11:40:49.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:49.308979 19265 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-1/data,/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-1/wal
11:40:49.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c716745b4a1b4b39957ab73274b4df06"
11:40:49.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 11:40:42 on dist-test-slave-h1fj"
11:40:49.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:49.311305 19265 fs_report.cc:383] FS layout report
11:40:49.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
11:40:49.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-1/wal
11:40:49.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-1/wal
11:40:49.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-1/data/data
11:40:49.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
11:40:49.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
11:40:49.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
11:40:49.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
11:40:49.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
11:40:49.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
11:40:49.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
11:40:49.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
11:40:49.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
11:40:49.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
11:40:49.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
11:40:49.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:49.350221 19265 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
11:40:49.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:49.355857 19265 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
11:40:49.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:49.356298 19265 kserver.cc:163] Server-wide thread pool size limit: 3276
11:40:49.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:49.437359 19265 rpc_server.cc:307] RPC server started. Bound to: 127.18.108.125:36939
11:40:49.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:49.437451 19356 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.18.108.125:36939 every 8 connection(s)
11:40:49.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:49.441346 19265 server_base.cc:1166] Dumped server information to /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-1/data/info.pb
11:40:49.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:49.442528 18865 external_mini_cluster.cc:1388] Started /tmp/dist-test-task4Z5jUl/build/tsan/bin/kudu as pid 19265
11:40:49.443 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
11:40:49.443 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.18.108.124:42121
11:40:49.443 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
11:40:49.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:49.445212 18865 external_mini_cluster.cc:1326] Running /tmp/dist-test-task4Z5jUl/build/tsan/bin/kudu
11:40:49.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task4Z5jUl/build/tsan/bin/kudu
11:40:49.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-2/wal
11:40:49.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-2/data
11:40:49.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
11:40:49.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
11:40:49.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
11:40:49.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:40:49.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:40:49.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:40:49.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-2/logs
11:40:49.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-2/data/info.pb
11:40:49.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:40:49.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
11:40:49.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
11:40:49.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
11:40:49.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
11:40:49.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:40:49.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:40:49.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:40:49.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
11:40:49.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
11:40:49.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
11:40:49.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
11:40:49.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.108.124:42121
11:40:49.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.108.124
11:40:49.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37519
11:40:49.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/krb5kdc/kudu_127.18.108.124.keytab
11:40:49.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.18.108.124
11:40:49.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
11:40:49.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
11:40:49.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
11:40:49.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.108.84:34171
11:40:49.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:40:49.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:40:49.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:40:49.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
11:40:49.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.18.108.126:43211,127.18.108.125:36939,127.18.108.124:42121
11:40:49.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
11:40:49.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45823/jwks.json
11:40:49.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
11:40:49.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:49.470866 19357 sys_catalog.cc:263] Verifying existing consensus state
11:40:49.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:49.478189 19357 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06: Bootstrap starting.
11:40:49.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:49.525136 19357 log.cc:824] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06: Log is configured to *not* fsync() on all Append() calls
11:40:49.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:49.543514 19357 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
11:40:49.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:49.544390 19357 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06: Bootstrap complete.
11:40:49.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:49.578713 19357 raft_consensus.cc:348] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eb8f4e2d825e4e319fd704623a2750bf" member_type: VOTER last_known_addr { host: "127.18.108.126" port: 43211 } } peers { permanent_uuid: "c716745b4a1b4b39957ab73274b4df06" member_type: VOTER last_known_addr { host: "127.18.108.125" port: 36939 } } peers { permanent_uuid: "5a5e3302510a40e28f4aa8f781a651ea" member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 } }
11:40:49.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:49.582002 19357 raft_consensus.cc:729] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: c716745b4a1b4b39957ab73274b4df06, State: Initialized, Role: FOLLOWER
11:40:49.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:49.582937 19357 consensus_queue.cc:260] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eb8f4e2d825e4e319fd704623a2750bf" member_type: VOTER last_known_addr { host: "127.18.108.126" port: 43211 } } peers { permanent_uuid: "c716745b4a1b4b39957ab73274b4df06" member_type: VOTER last_known_addr { host: "127.18.108.125" port: 36939 } } peers { permanent_uuid: "5a5e3302510a40e28f4aa8f781a651ea" member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 } }
11:40:49.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:49.585762 19361 sys_catalog.cc:455] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eb8f4e2d825e4e319fd704623a2750bf" member_type: VOTER last_known_addr { host: "127.18.108.126" port: 43211 } } peers { permanent_uuid: "c716745b4a1b4b39957ab73274b4df06" member_type: VOTER last_known_addr { host: "127.18.108.125" port: 36939 } } peers { permanent_uuid: "5a5e3302510a40e28f4aa8f781a651ea" member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 } } }
11:40:49.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:49.587532 19361 sys_catalog.cc:458] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [sys.catalog]: This master's current role is: FOLLOWER
11:40:49.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:49.587610 19357 sys_catalog.cc:564] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [sys.catalog]: configured and running, proceeding with master startup.
11:40:49.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:49.641880 19372 catalog_manager.cc:1259] Loaded cluster ID: 09fd2865e33b46ce9a354093f255891e
11:40:49.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:49.644057 19372 catalog_manager.cc:1552] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06: loading cluster ID for follower catalog manager: success
11:40:49.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 11:40:49 dist-test-slave-h1fj krb5kdc[18876](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790682049, etypes {rep=17 tkt=17 ses=17}, kudu/127.18.108.125@KRBTEST.COM for kudu/127.18.108.126@KRBTEST.COM
11:40:49.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:49.658237 19372 catalog_manager.cc:1574] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06: acquiring CA information for follower catalog manager: success
11:40:49.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:49.663486 19372 catalog_manager.cc:1602] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
11:40:49.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:49.707406 19265 master_runner.cc:186] Error getting master registration for 127.18.108.124:42121: Network error: Client connection negotiation failed: client connection to 127.18.108.124:42121: connect: Connection refused (error 111),
11:40:49.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:49.740568 19265 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
11:40:49.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:49.899776 19358 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
11:40:49.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:49.900312 19358 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
11:40:49.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:49.900879 19358 flags.cc:425] Enabled unsafe flag: --never_fsync=true
11:40:49.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:49.933072 19358 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
11:40:49.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:49.933281 19358 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
11:40:49.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:49.933586 19358 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
11:40:49.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:49.933780 19358 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
11:40:49.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:49.933966 19358 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
11:40:49.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:49.934144 19358 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
11:40:49.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:49.934280 19358 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:45823/jwks.json
11:40:49.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:49.970050 19358 master_runner.cc:386] Master server non-default flags:
11:40:49.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:40:49.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.108.84:34171
11:40:49.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:40:49.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:40:49.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-2/data
11:40:49.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-2/wal
11:40:49.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
11:40:49.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
11:40:49.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.18.108.126:43211,127.18.108.125:36939,127.18.108.124:42121
11:40:49.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/krb5kdc/kudu_127.18.108.124.keytab
11:40:49.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.18.108.124
11:40:49.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:40:49.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:40:49.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
11:40:49.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.108.124:42121
11:40:49.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
11:40:49.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
11:40:49.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
11:40:49.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45823/jwks.json
11:40:49.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
11:40:49.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
11:40:49.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
11:40:49.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:40:49.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:40:49.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-2/data/info.pb
11:40:49.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.108.124
11:40:49.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37519
11:40:49.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
11:40:49.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:40:49.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
11:40:49.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
11:40:49.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:40:49.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-2/logs
11:40:49.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:40:49.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
11:40:49.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
11:40:49.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
11:40:49.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
11:40:49.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
11:40:49.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
11:40:49.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 10:52:50 UTC on ccycloud.quasar-peqjre.root.comops.site
11:40:49.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679900
11:40:49.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
11:40:49.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:49.971208 19358 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
11:40:49.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:49.972357 19358 file_cache.cc:493] Constructed file cache file cache with capacity 419430
11:40:49.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:49.990971 19383 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:40:51.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:51.116680 19388 raft_consensus.cc:482] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
11:40:51.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:51.117333 19388 raft_consensus.cc:504] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eb8f4e2d825e4e319fd704623a2750bf" member_type: VOTER last_known_addr { host: "127.18.108.126" port: 43211 } } peers { permanent_uuid: "c716745b4a1b4b39957ab73274b4df06" member_type: VOTER last_known_addr { host: "127.18.108.125" port: 36939 } } peers { permanent_uuid: "5a5e3302510a40e28f4aa8f781a651ea" member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 } }
11:40:51.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:51.120788 19238 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "c716745b4a1b4b39957ab73274b4df06" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "eb8f4e2d825e4e319fd704623a2750bf" is_pre_election: true
11:40:51.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:51.121582 19238 raft_consensus.cc:2454] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c716745b4a1b4b39957ab73274b4df06 in term 1.
11:40:51.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:51.123283 19305 leader_election.cc:302] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: c716745b4a1b4b39957ab73274b4df06, eb8f4e2d825e4e319fd704623a2750bf; no voters:
11:40:51.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:51.125859 19388 leader_election.cc:288] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers eb8f4e2d825e4e319fd704623a2750bf (127.18.108.126:43211), 5a5e3302510a40e28f4aa8f781a651ea (127.18.108.124:42121)
11:40:51.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:51.126343 19388 raft_consensus.cc:2781] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [term 1 FOLLOWER]: Leader pre-election won for term 2
11:40:51.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:51.126650 19388 raft_consensus.cc:482] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [term 1 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
11:40:51.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:51.126961 19388 raft_consensus.cc:3037] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [term 1 FOLLOWER]: Advancing to term 2
11:40:51.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:51.136332 19307 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.18.108.124:42121: connect: Connection refused (error 111)
11:40:51.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:51.142819 19388 raft_consensus.cc:504] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eb8f4e2d825e4e319fd704623a2750bf" member_type: VOTER last_known_addr { host: "127.18.108.126" port: 43211 } } peers { permanent_uuid: "c716745b4a1b4b39957ab73274b4df06" member_type: VOTER last_known_addr { host: "127.18.108.125" port: 36939 } } peers { permanent_uuid: "5a5e3302510a40e28f4aa8f781a651ea" member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 } }
11:40:51.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:51.144490 19388 leader_election.cc:288] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [CANDIDATE]: Term 2 election: Requested vote from peers eb8f4e2d825e4e319fd704623a2750bf (127.18.108.126:43211), 5a5e3302510a40e28f4aa8f781a651ea (127.18.108.124:42121)
11:40:51.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:51.145260 19238 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "c716745b4a1b4b39957ab73274b4df06" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "eb8f4e2d825e4e319fd704623a2750bf"
11:40:51.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:51.145864 19238 raft_consensus.cc:3037] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [term 1 FOLLOWER]: Advancing to term 2
11:40:51.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:51.153606 19307 leader_election.cc:334] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 5a5e3302510a40e28f4aa8f781a651ea (127.18.108.124:42121): Network error: Client connection negotiation failed: client connection to 127.18.108.124:42121: connect: Connection refused (error 111)
11:40:51.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:51.156167 19238 raft_consensus.cc:2454] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c716745b4a1b4b39957ab73274b4df06 in term 2.
11:40:51.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:51.156457 19307 leader_election.cc:334] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 5a5e3302510a40e28f4aa8f781a651ea (127.18.108.124:42121): Network error: Client connection negotiation failed: client connection to 127.18.108.124:42121: connect: Connection refused (error 111)
11:40:51.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:51.157665 19305 leader_election.cc:302] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: c716745b4a1b4b39957ab73274b4df06, eb8f4e2d825e4e319fd704623a2750bf; no voters: 5a5e3302510a40e28f4aa8f781a651ea
11:40:51.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:51.158245 19388 raft_consensus.cc:2781] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [term 2 FOLLOWER]: Leader election won for term 2
11:40:51.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:51.158648 19388 raft_consensus.cc:686] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [term 2 LEADER]: Becoming Leader. State: Replica: c716745b4a1b4b39957ab73274b4df06, State: Running, Role: LEADER
11:40:51.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:51.159425 19388 consensus_queue.cc:237] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eb8f4e2d825e4e319fd704623a2750bf" member_type: VOTER last_known_addr { host: "127.18.108.126" port: 43211 } } peers { permanent_uuid: "c716745b4a1b4b39957ab73274b4df06" member_type: VOTER last_known_addr { host: "127.18.108.125" port: 36939 } } peers { permanent_uuid: "5a5e3302510a40e28f4aa8f781a651ea" member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 } }
11:40:51.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:51.167346 19390 sys_catalog.cc:455] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [sys.catalog]: SysCatalogTable state changed. Reason: New leader c716745b4a1b4b39957ab73274b4df06. Latest consensus state: current_term: 2 leader_uuid: "c716745b4a1b4b39957ab73274b4df06" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eb8f4e2d825e4e319fd704623a2750bf" member_type: VOTER last_known_addr { host: "127.18.108.126" port: 43211 } } peers { permanent_uuid: "c716745b4a1b4b39957ab73274b4df06" member_type: VOTER last_known_addr { host: "127.18.108.125" port: 36939 } } peers { permanent_uuid: "5a5e3302510a40e28f4aa8f781a651ea" member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 } } }
11:40:51.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:51.168308 19390 sys_catalog.cc:458] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [sys.catalog]: This master's current role is: LEADER
11:40:51.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:51.170859 19393 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
11:40:51.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:51.176329 19393 catalog_manager.cc:1484] Initializing Kudu cluster ID...
11:40:51.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:51.179509 19393 catalog_manager.cc:1259] Loaded cluster ID: 09fd2865e33b46ce9a354093f255891e
11:40:51.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:51.179684 19393 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
11:40:51.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:51.182866 19393 catalog_manager.cc:1504] Loading token signing keys...
11:40:51.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:51.185829 19393 catalog_manager.cc:5879] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06: Loaded TSK: 0
11:40:51.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:51.187158 19393 catalog_manager.cc:1514] Initializing in-progress tserver states...
11:40:51.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:49.991796 19385 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:40:51.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:49.991344 19382 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:40:51.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:51.227588 19384 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
11:40:51.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:51.227692 19358 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
11:40:51.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:51.231891 19358 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
11:40:51.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:51.234723 19358 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
11:40:51.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:51.236157 19358 hybrid_clock.cc:648] HybridClock initialized: now 1790682051236115 us; error 34 us; skew 500 ppm
11:40:51.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:51.236894 19358 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
11:40:51.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 11:40:51 dist-test-slave-h1fj krb5kdc[18876](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790682051, etypes {rep=17 tkt=17 ses=17}, kudu/127.18.108.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
11:40:51.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:51.242314 19358 init.cc:377] Logged in from keytab as kudu/127.18.108.124@KRBTEST.COM (short username kudu)
11:40:51.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:51.247839 19358 webserver.cc:460] Webserver started at http://127.18.108.124:37519/ using document root <none> and password file <none>
11:40:51.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:51.248939 19358 fs_manager.cc:359] Metadata directory not provided
11:40:51.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:51.249190 19358 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
11:40:51.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:51.257306 19358 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.000s sys 0.005s
11:40:51.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:51.261675 19399 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
11:40:51.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:51.262676 19358 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
11:40:51.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:51.262935 19358 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-2/data,/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-2/wal
11:40:51.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5a5e3302510a40e28f4aa8f781a651ea"
11:40:51.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 11:40:44 on dist-test-slave-h1fj"
11:40:51.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:51.265261 19358 fs_report.cc:383] FS layout report
11:40:51.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
11:40:51.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-2/wal
11:40:51.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-2/wal
11:40:51.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-2/data/data
11:40:51.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
11:40:51.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
11:40:51.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
11:40:51.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
11:40:51.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
11:40:51.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
11:40:51.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
11:40:51.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
11:40:51.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
11:40:51.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
11:40:51.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
11:40:51.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:51.305513 19358 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
11:40:51.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:51.310843 19358 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
11:40:51.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:51.311313 19358 kserver.cc:163] Server-wide thread pool size limit: 3276
11:40:51.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:51.393128 19358 rpc_server.cc:307] RPC server started. Bound to: 127.18.108.124:42121
11:40:51.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:51.393272 19450 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.18.108.124:42121 every 8 connection(s)
11:40:51.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:51.395958 19358 server_base.cc:1166] Dumped server information to /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-2/data/info.pb
11:40:51.398 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
11:40:51.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:51.397262 18865 external_mini_cluster.cc:1388] Started /tmp/dist-test-task4Z5jUl/build/tsan/bin/kudu as pid 19358
11:40:51.399 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.18.108.126:43211(127.18.108.126:43211)] connecting to peer
11:40:51.409 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.18.108.125:36939(127.18.108.125:36939)] connecting to peer
11:40:51.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:51.409675 19451 sys_catalog.cc:263] Verifying existing consensus state
11:40:51.411 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.18.108.124:42121(127.18.108.124:42121)] connecting to peer
11:40:51.418 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.18.108.126:43211(127.18.108.126:43211)] Successfully connected to peer
11:40:51.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:51.417546 19451 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5a5e3302510a40e28f4aa8f781a651ea: Bootstrap starting.
11:40:51.432 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.18.108.124:42121(127.18.108.124:42121)] Successfully connected to peer
11:40:51.435 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.18.108.125:36939(127.18.108.125:36939)] Successfully connected to peer
11:40:51.438 [DEBUG - kudu-nio-0] (SecurityContext.java:225) Refreshing Kerberos credentials...
11:40:51.439 [DEBUG - kudu-nio-0] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/krb5kdc/krb5cc
11:40:51.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 11:40:51 dist-test-slave-h1fj krb5kdc[18876](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790682044, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
11:40:51.455 [DEBUG - kudu-nio-0] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
11:40:51.463 [INFO - kudu-nio-0] (SecurityContext.java:259) Successfully refreshed Kerberos credentials from ticket cache
11:40:51.467 [DEBUG - kudu-nio-1] (SecurityContext.java:225) Refreshing Kerberos credentials...
11:40:51.467 [DEBUG - kudu-nio-1] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/krb5kdc/krb5cc
11:40:51.480 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.18.108.126
11:40:51.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 11:40:51 dist-test-slave-h1fj krb5kdc[18876](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790682044, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
11:40:51.490 [DEBUG - kudu-nio-1] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
11:40:51.494 [INFO - kudu-nio-1] (SecurityContext.java:259) Successfully refreshed Kerberos credentials from ticket cache
11:40:51.501 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.18.108.125
11:40:51.503 [DEBUG - kudu-nio-0] (SecurityContext.java:225) Refreshing Kerberos credentials...
11:40:51.503 [DEBUG - kudu-nio-0] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/krb5kdc/krb5cc
11:40:51.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 11:40:51 dist-test-slave-h1fj krb5kdc[18876](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790682044, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
11:40:51.508 [DEBUG - kudu-nio-0] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
11:40:51.515 [INFO - kudu-nio-0] (SecurityContext.java:259) Successfully refreshed Kerberos credentials from ticket cache
11:40:51.517 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.18.108.124
11:40:51.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:51.518832 19451 log.cc:824] T 00000000000000000000000000000000 P 5a5e3302510a40e28f4aa8f781a651ea: Log is configured to *not* fsync() on all Append() calls
11:40:51.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:51.552845 19451 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5a5e3302510a40e28f4aa8f781a651ea: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
11:40:51.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:51.553697 19451 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5a5e3302510a40e28f4aa8f781a651ea: Bootstrap complete.
11:40:51.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 11:40:51 dist-test-slave-h1fj krb5kdc[18876](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790682044, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.18.108.125@KRBTEST.COM
11:40:51.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:51.550933 18292 debug-util.cc:398] Leaking SignalData structure 0x7b0800076700 after lost signal to thread 18227
11:40:51.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:51.551618 18292 debug-util.cc:398] Leaking SignalData structure 0x7b0800081700 after lost signal to thread 18295
11:40:51.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:51.582688 19451 raft_consensus.cc:348] T 00000000000000000000000000000000 P 5a5e3302510a40e28f4aa8f781a651ea [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eb8f4e2d825e4e319fd704623a2750bf" member_type: VOTER last_known_addr { host: "127.18.108.126" port: 43211 } } peers { permanent_uuid: "c716745b4a1b4b39957ab73274b4df06" member_type: VOTER last_known_addr { host: "127.18.108.125" port: 36939 } } peers { permanent_uuid: "5a5e3302510a40e28f4aa8f781a651ea" member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 } }
11:40:51.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:51.586040 19451 raft_consensus.cc:729] T 00000000000000000000000000000000 P 5a5e3302510a40e28f4aa8f781a651ea [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5a5e3302510a40e28f4aa8f781a651ea, State: Initialized, Role: FOLLOWER
11:40:51.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:51.587028 19451 consensus_queue.cc:260] T 00000000000000000000000000000000 P 5a5e3302510a40e28f4aa8f781a651ea [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eb8f4e2d825e4e319fd704623a2750bf" member_type: VOTER last_known_addr { host: "127.18.108.126" port: 43211 } } peers { permanent_uuid: "c716745b4a1b4b39957ab73274b4df06" member_type: VOTER last_known_addr { host: "127.18.108.125" port: 36939 } } peers { permanent_uuid: "5a5e3302510a40e28f4aa8f781a651ea" member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 } }
11:40:51.591 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0xf45abd6c, L:/127.0.0.1:54972 - R:/127.18.108.125:36939] using SASL/GSSAPI
11:40:51.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 11:40:51 dist-test-slave-h1fj krb5kdc[18876](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790682044, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.18.108.124@KRBTEST.COM
11:40:51.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:51.608707 19460 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5a5e3302510a40e28f4aa8f781a651ea [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eb8f4e2d825e4e319fd704623a2750bf" member_type: VOTER last_known_addr { host: "127.18.108.126" port: 43211 } } peers { permanent_uuid: "c716745b4a1b4b39957ab73274b4df06" member_type: VOTER last_known_addr { host: "127.18.108.125" port: 36939 } } peers { permanent_uuid: "5a5e3302510a40e28f4aa8f781a651ea" member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 } } }
11:40:51.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:51.609584 19460 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5a5e3302510a40e28f4aa8f781a651ea [sys.catalog]: This master's current role is: FOLLOWER
11:40:51.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:51.610399 19451 sys_catalog.cc:564] T 00000000000000000000000000000000 P 5a5e3302510a40e28f4aa8f781a651ea [sys.catalog]: configured and running, proceeding with master startup.
11:40:51.614 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
11:40:51.614 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=ListTabletServers, tablet=null, attempt=1, TimeoutTracker(timeout=30000, elapsed=216), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.108.126:43211, [11ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.108.125:36939, [13ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.108.124:42121, [202ms] Sub RPC ConnectToMaster: received response from server master-127.18.108.125:36939: OK, deferred=null) after lookup
11:40:51.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 11:40:51 dist-test-slave-h1fj krb5kdc[18876](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790682044, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.18.108.126@KRBTEST.COM
11:40:51.648 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xe5a29425, L:/127.0.0.1:56234 - R:/127.18.108.124:42121] using SASL/GSSAPI
11:40:51.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 11:40:51 dist-test-slave-h1fj krb5kdc[18876](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790682049, etypes {rep=17 tkt=17 ses=17}, kudu/127.18.108.125@KRBTEST.COM for kudu/127.18.108.124@KRBTEST.COM
11:40:51.660 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xf0ed345b, L:/127.0.0.1:49222 - R:/127.18.108.126:43211] using SASL/GSSAPI
11:40:51.661 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.18.108.124:42121: Catalog manager is not initialized. State: Starting
11:40:51.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:51.683740 19426 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 5a5e3302510a40e28f4aa8f781a651ea [term 1 FOLLOWER]: Advancing to term 2
11:40:51.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:51.696280 19426 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 5a5e3302510a40e28f4aa8f781a651ea [term 2 FOLLOWER]: Refusing update from remote peer c716745b4a1b4b39957ab73274b4df06: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 2 index: 5. (index mismatch)
11:40:51.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:51.706679 19390 consensus_queue.cc:1035] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5a5e3302510a40e28f4aa8f781a651ea" member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.001s
11:40:51.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:51.717777 19472 catalog_manager.cc:795] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
11:40:51.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:51.721851 19238 raft_consensus.cc:1261] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [term 2 FOLLOWER]: Refusing update from remote peer c716745b4a1b4b39957ab73274b4df06: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 2 index: 5. (index mismatch)
11:40:51.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:51.724050 19390 consensus_queue.cc:1035] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [LEADER]: Connected to new peer: Peer: permanent_uuid: "eb8f4e2d825e4e319fd704623a2750bf" member_type: VOTER last_known_addr { host: "127.18.108.126" port: 43211 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
11:40:51.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:51.764052 19460 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5a5e3302510a40e28f4aa8f781a651ea [sys.catalog]: SysCatalogTable state changed. Reason: New leader c716745b4a1b4b39957ab73274b4df06. Latest consensus state: current_term: 2 leader_uuid: "c716745b4a1b4b39957ab73274b4df06" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eb8f4e2d825e4e319fd704623a2750bf" member_type: VOTER last_known_addr { host: "127.18.108.126" port: 43211 } } peers { permanent_uuid: "c716745b4a1b4b39957ab73274b4df06" member_type: VOTER last_known_addr { host: "127.18.108.125" port: 36939 } } peers { permanent_uuid: "5a5e3302510a40e28f4aa8f781a651ea" member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 } } }
11:40:51.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:51.764827 19460 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5a5e3302510a40e28f4aa8f781a651ea [sys.catalog]: This master's current role is: FOLLOWER
11:40:51.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:51.753305 19475 sys_catalog.cc:455] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [sys.catalog]: SysCatalogTable state changed. Reason: New leader c716745b4a1b4b39957ab73274b4df06. Latest consensus state: current_term: 2 leader_uuid: "c716745b4a1b4b39957ab73274b4df06" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eb8f4e2d825e4e319fd704623a2750bf" member_type: VOTER last_known_addr { host: "127.18.108.126" port: 43211 } } peers { permanent_uuid: "c716745b4a1b4b39957ab73274b4df06" member_type: VOTER last_known_addr { host: "127.18.108.125" port: 36939 } } peers { permanent_uuid: "5a5e3302510a40e28f4aa8f781a651ea" member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 } } }
11:40:51.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:51.776655 19475 sys_catalog.cc:458] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [sys.catalog]: This master's current role is: FOLLOWER
11:40:51.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:51.780274 19475 sys_catalog.cc:455] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "c716745b4a1b4b39957ab73274b4df06" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eb8f4e2d825e4e319fd704623a2750bf" member_type: VOTER last_known_addr { host: "127.18.108.126" port: 43211 } } peers { permanent_uuid: "c716745b4a1b4b39957ab73274b4df06" member_type: VOTER last_known_addr { host: "127.18.108.125" port: 36939 } } peers { permanent_uuid: "5a5e3302510a40e28f4aa8f781a651ea" member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 } } }
11:40:51.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:51.781045 19475 sys_catalog.cc:458] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [sys.catalog]: This master's current role is: FOLLOWER
11:40:51.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:51.784438 19478 sys_catalog.cc:455] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "c716745b4a1b4b39957ab73274b4df06" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eb8f4e2d825e4e319fd704623a2750bf" member_type: VOTER last_known_addr { host: "127.18.108.126" port: 43211 } } peers { permanent_uuid: "c716745b4a1b4b39957ab73274b4df06" member_type: VOTER last_known_addr { host: "127.18.108.125" port: 36939 } } peers { permanent_uuid: "5a5e3302510a40e28f4aa8f781a651ea" member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 } } }
11:40:51.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:51.785127 19478 sys_catalog.cc:458] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [sys.catalog]: This master's current role is: LEADER
11:40:51.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:51.786653 19390 sys_catalog.cc:455] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "c716745b4a1b4b39957ab73274b4df06" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eb8f4e2d825e4e319fd704623a2750bf" member_type: VOTER last_known_addr { host: "127.18.108.126" port: 43211 } } peers { permanent_uuid: "c716745b4a1b4b39957ab73274b4df06" member_type: VOTER last_known_addr { host: "127.18.108.125" port: 36939 } } peers { permanent_uuid: "5a5e3302510a40e28f4aa8f781a651ea" member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 } } }
11:40:51.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:51.787358 19390 sys_catalog.cc:458] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [sys.catalog]: This master's current role is: LEADER
11:40:51.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:51.790030 19460 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5a5e3302510a40e28f4aa8f781a651ea [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "c716745b4a1b4b39957ab73274b4df06" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eb8f4e2d825e4e319fd704623a2750bf" member_type: VOTER last_known_addr { host: "127.18.108.126" port: 43211 } } peers { permanent_uuid: "c716745b4a1b4b39957ab73274b4df06" member_type: VOTER last_known_addr { host: "127.18.108.125" port: 36939 } } peers { permanent_uuid: "5a5e3302510a40e28f4aa8f781a651ea" member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 } } }
11:40:51.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:51.790695 19460 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5a5e3302510a40e28f4aa8f781a651ea [sys.catalog]: This master's current role is: FOLLOWER
11:40:51.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 11:40:51 dist-test-slave-h1fj krb5kdc[18876](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790682051, etypes {rep=17 tkt=17 ses=17}, kudu/127.18.108.124@KRBTEST.COM for kudu/127.18.108.126@KRBTEST.COM
11:40:51.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 11:40:51 dist-test-slave-h1fj krb5kdc[18876](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790682051, etypes {rep=17 tkt=17 ses=17}, kudu/127.18.108.124@KRBTEST.COM for kudu/127.18.108.125@KRBTEST.COM
11:40:51.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 11:40:51 dist-test-slave-h1fj krb5kdc[18876](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790682051, etypes {rep=17 tkt=17 ses=17}, kudu/127.18.108.124@KRBTEST.COM for kudu/127.18.108.124@KRBTEST.COM
11:40:52.618 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.18.108.126:43211
11:40:52.618 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
11:40:52.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:52.619067 18865 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task4Z5jUl/build/tsan/bin/kudu with pid 19192
11:40:52.641 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.18.108.126:43211(127.18.108.126:43211)] handling channelInactive
11:40:52.641 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.18.108.126:43211(127.18.108.126:43211)] cleaning up while in state READY due to: connection closed
11:40:52.641 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
11:40:52.641 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.18.108.125:36939
11:40:52.641 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
11:40:52.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:52.642061 18865 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task4Z5jUl/build/tsan/bin/kudu with pid 19265
11:40:52.670 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.18.108.125:36939(127.18.108.125:36939)] handling channelInactive
11:40:52.670 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.18.108.125:36939(127.18.108.125:36939)] cleaning up while in state READY due to: connection closed
11:40:52.671 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
11:40:52.671 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.18.108.124:42121
11:40:52.671 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
11:40:52.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:52.671707 18865 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task4Z5jUl/build/tsan/bin/kudu with pid 19358
11:40:52.691 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.18.108.124:42121(127.18.108.124:42121)] handling channelInactive
11:40:52.691 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
11:40:52.691 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.18.108.124:42121(127.18.108.124:42121)] cleaning up while in state READY due to: connection closed
11:40:52.691 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.18.108.126:43211
11:40:52.692 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
11:40:52.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:52.693460 18865 external_mini_cluster.cc:1326] Running /tmp/dist-test-task4Z5jUl/build/tsan/bin/kudu
11:40:52.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task4Z5jUl/build/tsan/bin/kudu
11:40:52.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-0/wal
11:40:52.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-0/data
11:40:52.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
11:40:52.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
11:40:52.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
11:40:52.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:40:52.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:40:52.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:40:52.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-0/logs
11:40:52.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-0/data/info.pb
11:40:52.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:40:52.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
11:40:52.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
11:40:52.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
11:40:52.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
11:40:52.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:40:52.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:40:52.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:40:52.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
11:40:52.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
11:40:52.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
11:40:52.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
11:40:52.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.108.126:43211
11:40:52.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.108.126
11:40:52.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36579
11:40:52.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/krb5kdc/kudu_127.18.108.126.keytab
11:40:52.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.18.108.126
11:40:52.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
11:40:52.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
11:40:52.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
11:40:52.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.108.84:34171
11:40:52.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:40:52.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:40:52.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:40:52.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
11:40:52.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.18.108.126:43211,127.18.108.125:36939,127.18.108.124:42121
11:40:52.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
11:40:52.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45823/jwks.json
11:40:52.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
11:40:52.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:52.992439 19486 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
11:40:52.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:52.993058 19486 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
11:40:52.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:52.993716 19486 flags.cc:425] Enabled unsafe flag: --never_fsync=true
11:40:53.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:53.024271 19486 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
11:40:53.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:53.024509 19486 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
11:40:53.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:53.024932 19486 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
11:40:53.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:53.025229 19486 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
11:40:53.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:53.025489 19486 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
11:40:53.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:53.025753 19486 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
11:40:53.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:53.025946 19486 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:45823/jwks.json
11:40:53.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:53.061530 19486 master_runner.cc:386] Master server non-default flags:
11:40:53.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:40:53.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.108.84:34171
11:40:53.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:40:53.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:40:53.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-0/data
11:40:53.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-0/wal
11:40:53.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
11:40:53.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
11:40:53.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.18.108.126:43211,127.18.108.125:36939,127.18.108.124:42121
11:40:53.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/krb5kdc/kudu_127.18.108.126.keytab
11:40:53.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.18.108.126
11:40:53.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:40:53.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:40:53.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
11:40:53.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.108.126:43211
11:40:53.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
11:40:53.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
11:40:53.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
11:40:53.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45823/jwks.json
11:40:53.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
11:40:53.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
11:40:53.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
11:40:53.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:40:53.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:40:53.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-0/data/info.pb
11:40:53.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.108.126
11:40:53.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36579
11:40:53.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
11:40:53.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:40:53.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
11:40:53.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
11:40:53.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:40:53.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-0/logs
11:40:53.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:40:53.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
11:40:53.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
11:40:53.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
11:40:53.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
11:40:53.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
11:40:53.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
11:40:53.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 10:52:50 UTC on ccycloud.quasar-peqjre.root.comops.site
11:40:53.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679900
11:40:53.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
11:40:53.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:53.062920 19486 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
11:40:53.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:53.064026 19486 file_cache.cc:493] Constructed file cache file cache with capacity 419430
11:40:53.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:53.082590 19493 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:40:54.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:53.083415 19495 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:40:54.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:53.087121 19492 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:40:54.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:54.242115 19494 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1157 milliseconds
11:40:54.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:54.242240 19486 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
11:40:54.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:54.243500 19486 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
11:40:54.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:54.247658 19486 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
11:40:54.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:54.248989 19486 hybrid_clock.cc:648] HybridClock initialized: now 1790682054248956 us; error 45 us; skew 500 ppm
11:40:54.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:54.249754 19486 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
11:40:54.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 11:40:54 dist-test-slave-h1fj krb5kdc[18876](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790682054, etypes {rep=17 tkt=17 ses=17}, kudu/127.18.108.126@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
11:40:54.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:54.255411 19486 init.cc:377] Logged in from keytab as kudu/127.18.108.126@KRBTEST.COM (short username kudu)
11:40:54.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:54.261251 19486 webserver.cc:460] Webserver started at http://127.18.108.126:36579/ using document root <none> and password file <none>
11:40:54.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:54.262357 19486 fs_manager.cc:359] Metadata directory not provided
11:40:54.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:54.262601 19486 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
11:40:54.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:54.272277 19486 fs_manager.cc:709] Time spent opening directory manager: real 0.006s user 0.006s sys 0.001s
11:40:54.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:54.277638 19503 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
11:40:54.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:54.278885 19486 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.001s
11:40:54.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:54.279155 19486 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-0/data,/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-0/wal
11:40:54.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "eb8f4e2d825e4e319fd704623a2750bf"
11:40:54.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 11:40:40 on dist-test-slave-h1fj"
11:40:54.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:54.281435 19486 fs_report.cc:383] FS layout report
11:40:54.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
11:40:54.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-0/wal
11:40:54.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-0/wal
11:40:54.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-0/data/data
11:40:54.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
11:40:54.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
11:40:54.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
11:40:54.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
11:40:54.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
11:40:54.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
11:40:54.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
11:40:54.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
11:40:54.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
11:40:54.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
11:40:54.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
11:40:54.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:54.326382 19486 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
11:40:54.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:54.331590 19486 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
11:40:54.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:54.332020 19486 kserver.cc:163] Server-wide thread pool size limit: 3276
11:40:54.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:54.400520 19486 rpc_server.cc:307] RPC server started. Bound to: 127.18.108.126:43211
11:40:54.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:54.400655 19554 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.18.108.126:43211 every 8 connection(s)
11:40:54.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:54.403265 19486 server_base.cc:1166] Dumped server information to /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-0/data/info.pb
11:40:54.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:54.404682 18865 external_mini_cluster.cc:1388] Started /tmp/dist-test-task4Z5jUl/build/tsan/bin/kudu as pid 19486
11:40:54.405 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
11:40:54.405 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.18.108.125:36939
11:40:54.405 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
11:40:54.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:54.407337 18865 external_mini_cluster.cc:1326] Running /tmp/dist-test-task4Z5jUl/build/tsan/bin/kudu
11:40:54.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task4Z5jUl/build/tsan/bin/kudu
11:40:54.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-1/wal
11:40:54.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-1/data
11:40:54.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
11:40:54.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
11:40:54.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
11:40:54.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:40:54.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:40:54.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:40:54.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-1/logs
11:40:54.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-1/data/info.pb
11:40:54.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:40:54.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
11:40:54.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
11:40:54.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
11:40:54.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
11:40:54.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:40:54.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:40:54.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:40:54.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
11:40:54.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
11:40:54.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
11:40:54.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
11:40:54.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.108.125:36939
11:40:54.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.108.125
11:40:54.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38867
11:40:54.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/krb5kdc/kudu_127.18.108.125.keytab
11:40:54.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.18.108.125
11:40:54.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
11:40:54.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
11:40:54.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
11:40:54.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.108.84:34171
11:40:54.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:40:54.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:40:54.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:40:54.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
11:40:54.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.18.108.126:43211,127.18.108.125:36939,127.18.108.124:42121
11:40:54.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
11:40:54.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45823/jwks.json
11:40:54.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
11:40:54.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:54.414302 19555 sys_catalog.cc:263] Verifying existing consensus state
11:40:54.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:54.419719 19555 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf: Bootstrap starting.
11:40:54.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:54.453681 19555 log.cc:824] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf: Log is configured to *not* fsync() on all Append() calls
11:40:54.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:54.471146 19555 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
11:40:54.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:54.471896 19555 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf: Bootstrap complete.
11:40:54.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:54.496294 19555 raft_consensus.cc:348] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eb8f4e2d825e4e319fd704623a2750bf" member_type: VOTER last_known_addr { host: "127.18.108.126" port: 43211 } } peers { permanent_uuid: "c716745b4a1b4b39957ab73274b4df06" member_type: VOTER last_known_addr { host: "127.18.108.125" port: 36939 } } peers { permanent_uuid: "5a5e3302510a40e28f4aa8f781a651ea" member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 } }
11:40:54.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:54.498787 19555 raft_consensus.cc:729] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: eb8f4e2d825e4e319fd704623a2750bf, State: Initialized, Role: FOLLOWER
11:40:54.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:54.499615 19555 consensus_queue.cc:260] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 2.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eb8f4e2d825e4e319fd704623a2750bf" member_type: VOTER last_known_addr { host: "127.18.108.126" port: 43211 } } peers { permanent_uuid: "c716745b4a1b4b39957ab73274b4df06" member_type: VOTER last_known_addr { host: "127.18.108.125" port: 36939 } } peers { permanent_uuid: "5a5e3302510a40e28f4aa8f781a651ea" member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 } }
11:40:54.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:54.502235 19559 sys_catalog.cc:455] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 2 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eb8f4e2d825e4e319fd704623a2750bf" member_type: VOTER last_known_addr { host: "127.18.108.126" port: 43211 } } peers { permanent_uuid: "c716745b4a1b4b39957ab73274b4df06" member_type: VOTER last_known_addr { host: "127.18.108.125" port: 36939 } } peers { permanent_uuid: "5a5e3302510a40e28f4aa8f781a651ea" member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 } } }
11:40:54.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:54.503038 19559 sys_catalog.cc:458] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [sys.catalog]: This master's current role is: FOLLOWER
11:40:54.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:54.503863 19555 sys_catalog.cc:564] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [sys.catalog]: configured and running, proceeding with master startup.
11:40:54.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:54.540117 19570 catalog_manager.cc:1259] Loaded cluster ID: 09fd2865e33b46ce9a354093f255891e
11:40:54.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:54.540367 19570 catalog_manager.cc:1552] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf: loading cluster ID for follower catalog manager: success
11:40:54.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 11:40:54 dist-test-slave-h1fj krb5kdc[18876](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790682054, etypes {rep=17 tkt=17 ses=17}, kudu/127.18.108.126@KRBTEST.COM for kudu/127.18.108.126@KRBTEST.COM
11:40:54.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:54.549203 19570 catalog_manager.cc:1574] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf: acquiring CA information for follower catalog manager: success
11:40:54.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:54.554800 19570 catalog_manager.cc:1602] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
11:40:54.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:54.565582 19486 master_runner.cc:186] Error getting master registration for 127.18.108.125:36939: Network error: Client connection negotiation failed: client connection to 127.18.108.125:36939: connect: Connection refused (error 111),
11:40:54.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:54.584401 19486 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
11:40:54.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:54.776650 19557 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
11:40:54.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:54.777392 19557 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
11:40:54.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:54.778420 19557 flags.cc:425] Enabled unsafe flag: --never_fsync=true
11:40:54.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:54.821442 19557 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
11:40:54.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:54.821704 19557 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
11:40:54.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:54.822089 19557 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
11:40:54.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:54.822369 19557 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
11:40:54.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:54.822639 19557 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
11:40:54.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:54.822917 19557 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
11:40:54.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:54.823113 19557 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:45823/jwks.json
11:40:54.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:54.859210 19557 master_runner.cc:386] Master server non-default flags:
11:40:54.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:40:54.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.108.84:34171
11:40:54.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:40:54.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:40:54.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-1/data
11:40:54.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-1/wal
11:40:54.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
11:40:54.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
11:40:54.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.18.108.126:43211,127.18.108.125:36939,127.18.108.124:42121
11:40:54.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/krb5kdc/kudu_127.18.108.125.keytab
11:40:54.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.18.108.125
11:40:54.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:40:54.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:40:54.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
11:40:54.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.108.125:36939
11:40:54.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
11:40:54.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
11:40:54.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
11:40:54.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45823/jwks.json
11:40:54.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
11:40:54.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
11:40:54.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
11:40:54.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:40:54.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:40:54.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-1/data/info.pb
11:40:54.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.108.125
11:40:54.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38867
11:40:54.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
11:40:54.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:40:54.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
11:40:54.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
11:40:54.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:40:54.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-1/logs
11:40:54.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:40:54.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
11:40:54.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
11:40:54.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
11:40:54.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
11:40:54.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
11:40:54.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
11:40:54.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 10:52:50 UTC on ccycloud.quasar-peqjre.root.comops.site
11:40:54.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679900
11:40:54.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
11:40:54.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:54.860625 19557 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
11:40:54.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:54.861807 19557 file_cache.cc:493] Constructed file cache file cache with capacity 419430
11:40:54.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:54.879833 19581 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:40:55.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:55.755142 19586 raft_consensus.cc:482] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
11:40:55.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:55.755738 19586 raft_consensus.cc:504] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eb8f4e2d825e4e319fd704623a2750bf" member_type: VOTER last_known_addr { host: "127.18.108.126" port: 43211 } } peers { permanent_uuid: "c716745b4a1b4b39957ab73274b4df06" member_type: VOTER last_known_addr { host: "127.18.108.125" port: 36939 } } peers { permanent_uuid: "5a5e3302510a40e28f4aa8f781a651ea" member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 } }
11:40:55.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:55.759104 19586 leader_election.cc:288] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers c716745b4a1b4b39957ab73274b4df06 (127.18.108.125:36939), 5a5e3302510a40e28f4aa8f781a651ea (127.18.108.124:42121)
11:40:55.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:55.771500 19504 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.18.108.125:36939: connect: Connection refused (error 111)
11:40:55.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:55.776582 19504 leader_election.cc:334] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer c716745b4a1b4b39957ab73274b4df06 (127.18.108.125:36939): Network error: Client connection negotiation failed: client connection to 127.18.108.125:36939: connect: Connection refused (error 111)
11:40:55.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:55.777369 19506 leader_election.cc:334] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 5a5e3302510a40e28f4aa8f781a651ea (127.18.108.124:42121): Network error: Client connection negotiation failed: client connection to 127.18.108.124:42121: connect: Connection refused (error 111)
11:40:55.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:55.777812 19506 leader_election.cc:302] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: eb8f4e2d825e4e319fd704623a2750bf; no voters: 5a5e3302510a40e28f4aa8f781a651ea, c716745b4a1b4b39957ab73274b4df06
11:40:55.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:55.778664 19586 raft_consensus.cc:2726] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [term 2 FOLLOWER]: Leader pre-election lost for term 3. Reason: could not achieve majority
11:40:56.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:54.880033 19583 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:40:56.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:54.881105 19580 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:40:56.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:56.006721 19582 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
11:40:56.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:56.007267 19557 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
11:40:56.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:56.011672 19557 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
11:40:56.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:56.014739 19557 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
11:40:56.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:56.016168 19557 hybrid_clock.cc:648] HybridClock initialized: now 1790682056016099 us; error 39 us; skew 500 ppm
11:40:56.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:56.017323 19557 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
11:40:56.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 11:40:56 dist-test-slave-h1fj krb5kdc[18876](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790682056, etypes {rep=17 tkt=17 ses=17}, kudu/127.18.108.125@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
11:40:56.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:56.024019 19557 init.cc:377] Logged in from keytab as kudu/127.18.108.125@KRBTEST.COM (short username kudu)
11:40:56.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:56.028977 19557 webserver.cc:460] Webserver started at http://127.18.108.125:38867/ using document root <none> and password file <none>
11:40:56.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:56.029796 19557 fs_manager.cc:359] Metadata directory not provided
11:40:56.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:56.029968 19557 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
11:40:56.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:56.037789 19557 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.005s sys 0.000s
11:40:56.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:56.042372 19596 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
11:40:56.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:56.043380 19557 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
11:40:56.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:56.043646 19557 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-1/data,/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-1/wal
11:40:56.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c716745b4a1b4b39957ab73274b4df06"
11:40:56.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 11:40:42 on dist-test-slave-h1fj"
11:40:56.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:56.045826 19557 fs_report.cc:383] FS layout report
11:40:56.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
11:40:56.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-1/wal
11:40:56.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-1/wal
11:40:56.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-1/data/data
11:40:56.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
11:40:56.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
11:40:56.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
11:40:56.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
11:40:56.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
11:40:56.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
11:40:56.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
11:40:56.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
11:40:56.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
11:40:56.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
11:40:56.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
11:40:56.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:56.084010 19557 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
11:40:56.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:56.091111 19557 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
11:40:56.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:56.091675 19557 kserver.cc:163] Server-wide thread pool size limit: 3276
11:40:56.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:56.174937 19557 rpc_server.cc:307] RPC server started. Bound to: 127.18.108.125:36939
11:40:56.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:56.175055 19647 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.18.108.125:36939 every 8 connection(s)
11:40:56.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:56.177819 19557 server_base.cc:1166] Dumped server information to /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-1/data/info.pb
11:40:56.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:56.186789 18865 external_mini_cluster.cc:1388] Started /tmp/dist-test-task4Z5jUl/build/tsan/bin/kudu as pid 19557
11:40:56.187 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
11:40:56.187 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.18.108.124:42121
11:40:56.187 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
11:40:56.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:56.188768 19648 sys_catalog.cc:263] Verifying existing consensus state
11:40:56.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:56.189345 18865 external_mini_cluster.cc:1326] Running /tmp/dist-test-task4Z5jUl/build/tsan/bin/kudu
11:40:56.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task4Z5jUl/build/tsan/bin/kudu
11:40:56.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-2/wal
11:40:56.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-2/data
11:40:56.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
11:40:56.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
11:40:56.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
11:40:56.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:40:56.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:40:56.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:40:56.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-2/logs
11:40:56.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-2/data/info.pb
11:40:56.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:40:56.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
11:40:56.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
11:40:56.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
11:40:56.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
11:40:56.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:40:56.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:40:56.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:40:56.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
11:40:56.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
11:40:56.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
11:40:56.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
11:40:56.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.108.124:42121
11:40:56.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.108.124
11:40:56.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37519
11:40:56.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/krb5kdc/kudu_127.18.108.124.keytab
11:40:56.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.18.108.124
11:40:56.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
11:40:56.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
11:40:56.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
11:40:56.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.108.84:34171
11:40:56.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:40:56.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:40:56.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:40:56.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
11:40:56.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.18.108.126:43211,127.18.108.125:36939,127.18.108.124:42121
11:40:56.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
11:40:56.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45823/jwks.json
11:40:56.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
11:40:56.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:56.196419 19648 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06: Bootstrap starting.
11:40:56.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:56.247037 19648 log.cc:824] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06: Log is configured to *not* fsync() on all Append() calls
11:40:56.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:56.267123 19648 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
11:40:56.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:56.268041 19648 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06: Bootstrap complete.
11:40:56.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:56.314947 19648 raft_consensus.cc:348] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eb8f4e2d825e4e319fd704623a2750bf" member_type: VOTER last_known_addr { host: "127.18.108.126" port: 43211 } } peers { permanent_uuid: "c716745b4a1b4b39957ab73274b4df06" member_type: VOTER last_known_addr { host: "127.18.108.125" port: 36939 } } peers { permanent_uuid: "5a5e3302510a40e28f4aa8f781a651ea" member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 } }
11:40:56.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:56.318270 19648 raft_consensus.cc:729] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: c716745b4a1b4b39957ab73274b4df06, State: Initialized, Role: FOLLOWER
11:40:56.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:56.319177 19648 consensus_queue.cc:260] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 2.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eb8f4e2d825e4e319fd704623a2750bf" member_type: VOTER last_known_addr { host: "127.18.108.126" port: 43211 } } peers { permanent_uuid: "c716745b4a1b4b39957ab73274b4df06" member_type: VOTER last_known_addr { host: "127.18.108.125" port: 36939 } } peers { permanent_uuid: "5a5e3302510a40e28f4aa8f781a651ea" member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 } }
11:40:56.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:56.321915 19653 sys_catalog.cc:455] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 2 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eb8f4e2d825e4e319fd704623a2750bf" member_type: VOTER last_known_addr { host: "127.18.108.126" port: 43211 } } peers { permanent_uuid: "c716745b4a1b4b39957ab73274b4df06" member_type: VOTER last_known_addr { host: "127.18.108.125" port: 36939 } } peers { permanent_uuid: "5a5e3302510a40e28f4aa8f781a651ea" member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 } } }
11:40:56.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:56.322896 19653 sys_catalog.cc:458] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [sys.catalog]: This master's current role is: FOLLOWER
11:40:56.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:56.323686 19648 sys_catalog.cc:564] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [sys.catalog]: configured and running, proceeding with master startup.
11:40:56.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:56.354462 19664 catalog_manager.cc:795] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
11:40:56.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 11:40:56 dist-test-slave-h1fj krb5kdc[18876](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790682056, etypes {rep=17 tkt=17 ses=17}, kudu/127.18.108.125@KRBTEST.COM for kudu/127.18.108.126@KRBTEST.COM
11:40:56.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 11:40:56 dist-test-slave-h1fj krb5kdc[18876](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790682056, etypes {rep=17 tkt=17 ses=17}, kudu/127.18.108.125@KRBTEST.COM for kudu/127.18.108.125@KRBTEST.COM
11:40:56.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:56.448252 19557 master_runner.cc:186] Error getting master registration for 127.18.108.124:42121: Network error: Client connection negotiation failed: client connection to 127.18.108.124:42121: connect: Connection refused (error 111),
11:40:56.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:56.475421 19557 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
11:40:56.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:56.565434 19651 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
11:40:56.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:56.566011 19651 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
11:40:56.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:56.566787 19651 flags.cc:425] Enabled unsafe flag: --never_fsync=true
11:40:56.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:56.596808 19651 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
11:40:56.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:56.597038 19651 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
11:40:56.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:56.597405 19651 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
11:40:56.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:56.597679 19651 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
11:40:56.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:56.597976 19651 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
11:40:56.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:56.598258 19651 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
11:40:56.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:56.598418 19651 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:45823/jwks.json
11:40:56.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:56.634383 19651 master_runner.cc:386] Master server non-default flags:
11:40:56.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:40:56.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.108.84:34171
11:40:56.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:40:56.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:40:56.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-2/data
11:40:56.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-2/wal
11:40:56.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
11:40:56.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
11:40:56.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.18.108.126:43211,127.18.108.125:36939,127.18.108.124:42121
11:40:56.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/krb5kdc/kudu_127.18.108.124.keytab
11:40:56.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.18.108.124
11:40:56.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:40:56.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:40:56.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
11:40:56.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.108.124:42121
11:40:56.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
11:40:56.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
11:40:56.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
11:40:56.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45823/jwks.json
11:40:56.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
11:40:56.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
11:40:56.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
11:40:56.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:40:56.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:40:56.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-2/data/info.pb
11:40:56.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.108.124
11:40:56.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37519
11:40:56.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
11:40:56.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:40:56.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
11:40:56.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
11:40:56.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:40:56.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-2/logs
11:40:56.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:40:56.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
11:40:56.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
11:40:56.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
11:40:56.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
11:40:56.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
11:40:56.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
11:40:56.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 10:52:50 UTC on ccycloud.quasar-peqjre.root.comops.site
11:40:56.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679900
11:40:56.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
11:40:56.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:56.635484 19651 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
11:40:56.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:56.636408 19651 file_cache.cc:493] Constructed file cache file cache with capacity 419430
11:40:56.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:56.656107 19675 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:40:57.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:57.362563 19664 catalog_manager.cc:1259] Loaded cluster ID: 09fd2865e33b46ce9a354093f255891e
11:40:57.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:57.363085 19664 catalog_manager.cc:1552] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06: loading cluster ID for follower catalog manager: success
11:40:57.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:57.370687 19664 catalog_manager.cc:1574] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06: acquiring CA information for follower catalog manager: success
11:40:57.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:57.376088 19664 catalog_manager.cc:1602] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
11:40:57.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:57.700732 19681 raft_consensus.cc:482] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
11:40:57.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:57.701721 19681 raft_consensus.cc:504] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eb8f4e2d825e4e319fd704623a2750bf" member_type: VOTER last_known_addr { host: "127.18.108.126" port: 43211 } } peers { permanent_uuid: "c716745b4a1b4b39957ab73274b4df06" member_type: VOTER last_known_addr { host: "127.18.108.125" port: 36939 } } peers { permanent_uuid: "5a5e3302510a40e28f4aa8f781a651ea" member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 } }
11:40:57.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:57.706027 19530 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "c716745b4a1b4b39957ab73274b4df06" candidate_term: 3 candidate_status { last_received { term: 2 index: 5 } } ignore_live_leader: false dest_uuid: "eb8f4e2d825e4e319fd704623a2750bf" is_pre_election: true
11:40:57.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:57.706750 19530 raft_consensus.cc:2454] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c716745b4a1b4b39957ab73274b4df06 in term 2.
11:40:57.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:57.708117 19597 leader_election.cc:302] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: c716745b4a1b4b39957ab73274b4df06, eb8f4e2d825e4e319fd704623a2750bf; no voters:
11:40:57.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:57.712981 19682 raft_consensus.cc:2781] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [term 2 FOLLOWER]: Leader pre-election won for term 3
11:40:57.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:57.715992 19682 raft_consensus.cc:482] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [term 2 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
11:40:57.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:57.716311 19682 raft_consensus.cc:3037] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [term 2 FOLLOWER]: Advancing to term 3
11:40:57.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:57.719645 19681 leader_election.cc:288] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers eb8f4e2d825e4e319fd704623a2750bf (127.18.108.126:43211), 5a5e3302510a40e28f4aa8f781a651ea (127.18.108.124:42121)
11:40:57.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:57.724350 19599 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.18.108.124:42121: connect: Connection refused (error 111)
11:40:57.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:57.730557 19682 raft_consensus.cc:504] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eb8f4e2d825e4e319fd704623a2750bf" member_type: VOTER last_known_addr { host: "127.18.108.126" port: 43211 } } peers { permanent_uuid: "c716745b4a1b4b39957ab73274b4df06" member_type: VOTER last_known_addr { host: "127.18.108.125" port: 36939 } } peers { permanent_uuid: "5a5e3302510a40e28f4aa8f781a651ea" member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 } }
11:40:57.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:57.733708 19530 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "c716745b4a1b4b39957ab73274b4df06" candidate_term: 3 candidate_status { last_received { term: 2 index: 5 } } ignore_live_leader: false dest_uuid: "eb8f4e2d825e4e319fd704623a2750bf"
11:40:57.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:57.733922 19682 leader_election.cc:288] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [CANDIDATE]: Term 3 election: Requested vote from peers eb8f4e2d825e4e319fd704623a2750bf (127.18.108.126:43211), 5a5e3302510a40e28f4aa8f781a651ea (127.18.108.124:42121)
11:40:57.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:57.734223 19530 raft_consensus.cc:3037] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [term 2 FOLLOWER]: Advancing to term 3
11:40:57.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:57.738616 19599 leader_election.cc:334] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [CANDIDATE]: Term 3 election: RPC error from VoteRequest() call to peer 5a5e3302510a40e28f4aa8f781a651ea (127.18.108.124:42121): Network error: Client connection negotiation failed: client connection to 127.18.108.124:42121: connect: Connection refused (error 111)
11:40:57.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:57.739255 19599 leader_election.cc:334] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 5a5e3302510a40e28f4aa8f781a651ea (127.18.108.124:42121): Network error: Client connection negotiation failed: client connection to 127.18.108.124:42121: connect: Connection refused (error 111)
11:40:57.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:57.743882 19530 raft_consensus.cc:2454] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [term 3 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c716745b4a1b4b39957ab73274b4df06 in term 3.
11:40:57.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:57.744829 19597 leader_election.cc:302] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [CANDIDATE]: Term 3 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: c716745b4a1b4b39957ab73274b4df06, eb8f4e2d825e4e319fd704623a2750bf; no voters: 5a5e3302510a40e28f4aa8f781a651ea
11:40:57.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:57.745523 19682 raft_consensus.cc:2781] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [term 3 FOLLOWER]: Leader election won for term 3
11:40:57.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:57.745946 19682 raft_consensus.cc:686] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [term 3 LEADER]: Becoming Leader. State: Replica: c716745b4a1b4b39957ab73274b4df06, State: Running, Role: LEADER
11:40:57.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:57.746757 19682 consensus_queue.cc:237] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 2.5, Last appended by leader: 5, Current term: 3, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eb8f4e2d825e4e319fd704623a2750bf" member_type: VOTER last_known_addr { host: "127.18.108.126" port: 43211 } } peers { permanent_uuid: "c716745b4a1b4b39957ab73274b4df06" member_type: VOTER last_known_addr { host: "127.18.108.125" port: 36939 } } peers { permanent_uuid: "5a5e3302510a40e28f4aa8f781a651ea" member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 } }
11:40:57.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:57.756449 19681 sys_catalog.cc:455] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [sys.catalog]: SysCatalogTable state changed. Reason: New leader c716745b4a1b4b39957ab73274b4df06. Latest consensus state: current_term: 3 leader_uuid: "c716745b4a1b4b39957ab73274b4df06" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eb8f4e2d825e4e319fd704623a2750bf" member_type: VOTER last_known_addr { host: "127.18.108.126" port: 43211 } } peers { permanent_uuid: "c716745b4a1b4b39957ab73274b4df06" member_type: VOTER last_known_addr { host: "127.18.108.125" port: 36939 } } peers { permanent_uuid: "5a5e3302510a40e28f4aa8f781a651ea" member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 } } }
11:40:57.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:57.757136 19681 sys_catalog.cc:458] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [sys.catalog]: This master's current role is: LEADER
11:40:57.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:57.759547 19686 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
11:40:57.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:57.767899 19686 catalog_manager.cc:1484] Initializing Kudu cluster ID...
11:40:57.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:57.771927 19686 catalog_manager.cc:1259] Loaded cluster ID: 09fd2865e33b46ce9a354093f255891e
11:40:57.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:57.772140 19686 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
11:40:57.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:57.777586 19686 catalog_manager.cc:1504] Loading token signing keys...
11:40:57.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:57.781718 19686 catalog_manager.cc:5879] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06: Loaded TSK: 0
11:40:57.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:57.783164 19686 catalog_manager.cc:1514] Initializing in-progress tserver states...
11:40:57.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:56.656136 19674 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:40:57.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:56.657810 19651 server_base.cc:1034] running on GCE node
11:40:57.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:56.656303 19677 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:40:57.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:57.819687 19651 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
11:40:57.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:57.822817 19651 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
11:40:57.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:57.824312 19651 hybrid_clock.cc:648] HybridClock initialized: now 1790682057824282 us; error 41 us; skew 500 ppm
11:40:57.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:57.825338 19651 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
11:40:57.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 11:40:57 dist-test-slave-h1fj krb5kdc[18876](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790682057, etypes {rep=17 tkt=17 ses=17}, kudu/127.18.108.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
11:40:57.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:57.836424 19651 init.cc:377] Logged in from keytab as kudu/127.18.108.124@KRBTEST.COM (short username kudu)
11:40:57.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:57.841917 19651 webserver.cc:460] Webserver started at http://127.18.108.124:37519/ using document root <none> and password file <none>
11:40:57.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:57.843060 19651 fs_manager.cc:359] Metadata directory not provided
11:40:57.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:57.843344 19651 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
11:40:57.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:57.853576 19651 fs_manager.cc:709] Time spent opening directory manager: real 0.006s user 0.001s sys 0.004s
11:40:57.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:57.858771 19693 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
11:40:57.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:57.859952 19651 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
11:40:57.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:57.860342 19651 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-2/data,/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-2/wal
11:40:57.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5a5e3302510a40e28f4aa8f781a651ea"
11:40:57.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 11:40:44 on dist-test-slave-h1fj"
11:40:57.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:57.863332 19651 fs_report.cc:383] FS layout report
11:40:57.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
11:40:57.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-2/wal
11:40:57.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-2/wal
11:40:57.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-2/data/data
11:40:57.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
11:40:57.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
11:40:57.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
11:40:57.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
11:40:57.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
11:40:57.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
11:40:57.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
11:40:57.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
11:40:57.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
11:40:57.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
11:40:57.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
11:40:57.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:57.928802 19651 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
11:40:57.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:57.936561 19651 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
11:40:57.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:57.937090 19651 kserver.cc:163] Server-wide thread pool size limit: 3276
11:40:58.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:58.016434 19651 rpc_server.cc:307] RPC server started. Bound to: 127.18.108.124:42121
11:40:58.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:58.016516 19744 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.18.108.124:42121 every 8 connection(s)
11:40:58.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:58.020164 19651 server_base.cc:1166] Dumped server information to /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-2/data/info.pb
11:40:58.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:58.028957 18865 external_mini_cluster.cc:1388] Started /tmp/dist-test-task4Z5jUl/build/tsan/bin/kudu as pid 19651
11:40:58.029 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
11:40:58.030 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.18.108.125:36939(127.18.108.125:36939)] connecting to peer
11:40:58.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:58.031484 19745 sys_catalog.cc:263] Verifying existing consensus state
11:40:58.033 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.18.108.125:36939(127.18.108.125:36939)] Successfully connected to peer
11:40:58.036 [DEBUG - kudu-nio-1] (SecurityContext.java:225) Refreshing Kerberos credentials...
11:40:58.036 [DEBUG - kudu-nio-1] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/krb5kdc/krb5cc
11:40:58.040 [DEBUG - kudu-nio-1] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
11:40:58.040 [INFO - kudu-nio-1] (SecurityContext.java:259) Successfully refreshed Kerberos credentials from ticket cache
11:40:58.041 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.18.108.125
11:40:58.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:58.039000 19745 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5a5e3302510a40e28f4aa8f781a651ea: Bootstrap starting.
11:40:58.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 11:40:58 dist-test-slave-h1fj krb5kdc[18876](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790682044, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
11:40:58.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 11:40:58 dist-test-slave-h1fj krb5kdc[18876](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790682044, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.18.108.125@KRBTEST.COM
11:40:58.073 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x9d302d48, L:/127.0.0.1:55038 - R:/127.18.108.125:36939] using SASL/GSSAPI
11:40:58.075 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
11:40:58.075 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
11:40:58.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:58.086585 19745 log.cc:824] T 00000000000000000000000000000000 P 5a5e3302510a40e28f4aa8f781a651ea: Log is configured to *not* fsync() on all Append() calls
11:40:58.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 11:40:58 dist-test-slave-h1fj krb5kdc[18876](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790682058, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
11:40:58.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
11:40:58.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:58.124045 19745 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5a5e3302510a40e28f4aa8f781a651ea: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
11:40:58.128 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
11:40:58.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:58.128683 19745 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5a5e3302510a40e28f4aa8f781a651ea: Bootstrap complete.
11:40:58.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:58.162817 19745 raft_consensus.cc:348] T 00000000000000000000000000000000 P 5a5e3302510a40e28f4aa8f781a651ea [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eb8f4e2d825e4e319fd704623a2750bf" member_type: VOTER last_known_addr { host: "127.18.108.126" port: 43211 } } peers { permanent_uuid: "c716745b4a1b4b39957ab73274b4df06" member_type: VOTER last_known_addr { host: "127.18.108.125" port: 36939 } } peers { permanent_uuid: "5a5e3302510a40e28f4aa8f781a651ea" member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 } }
11:40:58.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:58.166011 19745 raft_consensus.cc:729] T 00000000000000000000000000000000 P 5a5e3302510a40e28f4aa8f781a651ea [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5a5e3302510a40e28f4aa8f781a651ea, State: Initialized, Role: FOLLOWER
11:40:58.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:58.166978 19745 consensus_queue.cc:260] T 00000000000000000000000000000000 P 5a5e3302510a40e28f4aa8f781a651ea [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 2.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eb8f4e2d825e4e319fd704623a2750bf" member_type: VOTER last_known_addr { host: "127.18.108.126" port: 43211 } } peers { permanent_uuid: "c716745b4a1b4b39957ab73274b4df06" member_type: VOTER last_known_addr { host: "127.18.108.125" port: 36939 } } peers { permanent_uuid: "5a5e3302510a40e28f4aa8f781a651ea" member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 } }
11:40:58.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:58.175220 19751 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5a5e3302510a40e28f4aa8f781a651ea [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 2 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eb8f4e2d825e4e319fd704623a2750bf" member_type: VOTER last_known_addr { host: "127.18.108.126" port: 43211 } } peers { permanent_uuid: "c716745b4a1b4b39957ab73274b4df06" member_type: VOTER last_known_addr { host: "127.18.108.125" port: 36939 } } peers { permanent_uuid: "5a5e3302510a40e28f4aa8f781a651ea" member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 } } }
11:40:58.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:58.176041 19751 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5a5e3302510a40e28f4aa8f781a651ea [sys.catalog]: This master's current role is: FOLLOWER
11:40:58.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:58.178634 19745 sys_catalog.cc:564] T 00000000000000000000000000000000 P 5a5e3302510a40e28f4aa8f781a651ea [sys.catalog]: configured and running, proceeding with master startup.
11:40:58.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:58.261752 19763 catalog_manager.cc:1259] Loaded cluster ID: 09fd2865e33b46ce9a354093f255891e
11:40:58.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:58.262089 19763 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 5a5e3302510a40e28f4aa8f781a651ea: loading cluster ID for follower catalog manager: success
11:40:58.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 11:40:58 dist-test-slave-h1fj krb5kdc[18876](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790682057, etypes {rep=17 tkt=17 ses=17}, kudu/127.18.108.124@KRBTEST.COM for kudu/127.18.108.126@KRBTEST.COM
11:40:58.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:58.280664 19763 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 5a5e3302510a40e28f4aa8f781a651ea: acquiring CA information for follower catalog manager: success
11:40:58.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:58.288020 19763 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 5a5e3302510a40e28f4aa8f781a651ea: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
11:40:58.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:58.342895 19720 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 5a5e3302510a40e28f4aa8f781a651ea [term 2 FOLLOWER]: Advancing to term 3
11:40:58.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:58.349519 19530 raft_consensus.cc:1261] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [term 3 FOLLOWER]: Refusing update from remote peer c716745b4a1b4b39957ab73274b4df06: Log matching property violated. Preceding OpId in replica: term: 2 index: 5. Preceding OpId from leader: term: 3 index: 6. (index mismatch)
11:40:58.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:58.352108 19767 consensus_queue.cc:1035] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [LEADER]: Connected to new peer: Peer: permanent_uuid: "eb8f4e2d825e4e319fd704623a2750bf" member_type: VOTER last_known_addr { host: "127.18.108.126" port: 43211 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
11:40:58.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:58.365468 19720 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 5a5e3302510a40e28f4aa8f781a651ea [term 3 FOLLOWER]: Refusing update from remote peer c716745b4a1b4b39957ab73274b4df06: Log matching property violated. Preceding OpId in replica: term: 2 index: 5. Preceding OpId from leader: term: 3 index: 6. (index mismatch)
11:40:58.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:58.367475 19767 consensus_queue.cc:1035] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5a5e3302510a40e28f4aa8f781a651ea" member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
11:40:58.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:58.385069 19771 sys_catalog.cc:455] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [sys.catalog]: SysCatalogTable state changed. Reason: New leader c716745b4a1b4b39957ab73274b4df06. Latest consensus state: current_term: 3 leader_uuid: "c716745b4a1b4b39957ab73274b4df06" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eb8f4e2d825e4e319fd704623a2750bf" member_type: VOTER last_known_addr { host: "127.18.108.126" port: 43211 } } peers { permanent_uuid: "c716745b4a1b4b39957ab73274b4df06" member_type: VOTER last_known_addr { host: "127.18.108.125" port: 36939 } } peers { permanent_uuid: "5a5e3302510a40e28f4aa8f781a651ea" member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 } } }
11:40:58.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:58.392211 19751 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5a5e3302510a40e28f4aa8f781a651ea [sys.catalog]: SysCatalogTable state changed. Reason: New leader c716745b4a1b4b39957ab73274b4df06. Latest consensus state: current_term: 3 leader_uuid: "c716745b4a1b4b39957ab73274b4df06" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eb8f4e2d825e4e319fd704623a2750bf" member_type: VOTER last_known_addr { host: "127.18.108.126" port: 43211 } } peers { permanent_uuid: "c716745b4a1b4b39957ab73274b4df06" member_type: VOTER last_known_addr { host: "127.18.108.125" port: 36939 } } peers { permanent_uuid: "5a5e3302510a40e28f4aa8f781a651ea" member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 } } }
11:40:58.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:58.392978 19751 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5a5e3302510a40e28f4aa8f781a651ea [sys.catalog]: This master's current role is: FOLLOWER
11:40:58.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:58.393875 19771 sys_catalog.cc:458] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [sys.catalog]: This master's current role is: FOLLOWER
11:40:58.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:58.433113 19771 sys_catalog.cc:455] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "c716745b4a1b4b39957ab73274b4df06" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eb8f4e2d825e4e319fd704623a2750bf" member_type: VOTER last_known_addr { host: "127.18.108.126" port: 43211 } } peers { permanent_uuid: "c716745b4a1b4b39957ab73274b4df06" member_type: VOTER last_known_addr { host: "127.18.108.125" port: 36939 } } peers { permanent_uuid: "5a5e3302510a40e28f4aa8f781a651ea" member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 } } }
11:40:58.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:58.434013 19771 sys_catalog.cc:458] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [sys.catalog]: This master's current role is: FOLLOWER
11:40:58.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:58.435336 19751 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5a5e3302510a40e28f4aa8f781a651ea [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "c716745b4a1b4b39957ab73274b4df06" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eb8f4e2d825e4e319fd704623a2750bf" member_type: VOTER last_known_addr { host: "127.18.108.126" port: 43211 } } peers { permanent_uuid: "c716745b4a1b4b39957ab73274b4df06" member_type: VOTER last_known_addr { host: "127.18.108.125" port: 36939 } } peers { permanent_uuid: "5a5e3302510a40e28f4aa8f781a651ea" member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 } } }
11:40:58.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:58.436036 19751 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5a5e3302510a40e28f4aa8f781a651ea [sys.catalog]: This master's current role is: FOLLOWER
11:40:58.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:58.438483 19774 sys_catalog.cc:455] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 3 leader_uuid: "c716745b4a1b4b39957ab73274b4df06" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eb8f4e2d825e4e319fd704623a2750bf" member_type: VOTER last_known_addr { host: "127.18.108.126" port: 43211 } } peers { permanent_uuid: "c716745b4a1b4b39957ab73274b4df06" member_type: VOTER last_known_addr { host: "127.18.108.125" port: 36939 } } peers { permanent_uuid: "5a5e3302510a40e28f4aa8f781a651ea" member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 } } }
11:40:58.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:58.440217 19774 sys_catalog.cc:458] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [sys.catalog]: This master's current role is: LEADER
11:40:58.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:58.439494 19767 sys_catalog.cc:455] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 3 leader_uuid: "c716745b4a1b4b39957ab73274b4df06" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eb8f4e2d825e4e319fd704623a2750bf" member_type: VOTER last_known_addr { host: "127.18.108.126" port: 43211 } } peers { permanent_uuid: "c716745b4a1b4b39957ab73274b4df06" member_type: VOTER last_known_addr { host: "127.18.108.125" port: 36939 } } peers { permanent_uuid: "5a5e3302510a40e28f4aa8f781a651ea" member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 } } }
11:40:58.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:58.441114 19767 sys_catalog.cc:458] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [sys.catalog]: This master's current role is: LEADER
11:40:59.128 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.18.108.126:43211
11:40:59.129 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
11:40:59.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:59.129617 18865 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task4Z5jUl/build/tsan/bin/kudu with pid 19486
11:40:59.166 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
11:40:59.166 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.18.108.125:36939
11:40:59.166 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
11:40:59.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:59.167059 18865 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task4Z5jUl/build/tsan/bin/kudu with pid 19557
11:40:59.189 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.18.108.125:36939(127.18.108.125:36939)] handling channelInactive
11:40:59.190 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
11:40:59.190 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.18.108.125:36939(127.18.108.125:36939)] cleaning up while in state READY due to: connection closed
11:40:59.190 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.18.108.124:42121
11:40:59.190 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
11:40:59.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:59.190598 18865 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task4Z5jUl/build/tsan/bin/kudu with pid 19651
11:40:59.209 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
11:40:59.209 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.18.108.126:43211
11:40:59.210 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
11:40:59.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:59.211433 18865 external_mini_cluster.cc:1326] Running /tmp/dist-test-task4Z5jUl/build/tsan/bin/kudu
11:40:59.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task4Z5jUl/build/tsan/bin/kudu
11:40:59.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-0/wal
11:40:59.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-0/data
11:40:59.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
11:40:59.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
11:40:59.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
11:40:59.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:40:59.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:40:59.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:40:59.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-0/logs
11:40:59.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-0/data/info.pb
11:40:59.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:40:59.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
11:40:59.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
11:40:59.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
11:40:59.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
11:40:59.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:40:59.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:40:59.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:40:59.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
11:40:59.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
11:40:59.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
11:40:59.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
11:40:59.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.108.126:43211
11:40:59.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.108.126
11:40:59.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36579
11:40:59.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/krb5kdc/kudu_127.18.108.126.keytab
11:40:59.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.18.108.126
11:40:59.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
11:40:59.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
11:40:59.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
11:40:59.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.108.84:34171
11:40:59.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:40:59.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:40:59.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:40:59.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
11:40:59.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.18.108.126:43211,127.18.108.125:36939,127.18.108.124:42121
11:40:59.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
11:40:59.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45823/jwks.json
11:40:59.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
11:40:59.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:59.513944 19781 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
11:40:59.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:59.514462 19781 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
11:40:59.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:59.515034 19781 flags.cc:425] Enabled unsafe flag: --never_fsync=true
11:40:59.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:59.545962 19781 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
11:40:59.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:59.546168 19781 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
11:40:59.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:59.546423 19781 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
11:40:59.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:59.546612 19781 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
11:40:59.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:59.546808 19781 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
11:40:59.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:59.546980 19781 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
11:40:59.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:59.547112 19781 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:45823/jwks.json
11:40:59.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:59.583176 19781 master_runner.cc:386] Master server non-default flags:
11:40:59.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:40:59.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.108.84:34171
11:40:59.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:40:59.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:40:59.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-0/data
11:40:59.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-0/wal
11:40:59.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
11:40:59.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
11:40:59.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.18.108.126:43211,127.18.108.125:36939,127.18.108.124:42121
11:40:59.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/krb5kdc/kudu_127.18.108.126.keytab
11:40:59.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.18.108.126
11:40:59.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:40:59.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:40:59.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
11:40:59.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.108.126:43211
11:40:59.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
11:40:59.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
11:40:59.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
11:40:59.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45823/jwks.json
11:40:59.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
11:40:59.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
11:40:59.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
11:40:59.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:40:59.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:40:59.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-0/data/info.pb
11:40:59.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.108.126
11:40:59.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36579
11:40:59.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
11:40:59.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:40:59.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
11:40:59.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
11:40:59.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:40:59.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-0/logs
11:40:59.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:40:59.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
11:40:59.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
11:40:59.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
11:40:59.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
11:40:59.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
11:40:59.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
11:40:59.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 10:52:50 UTC on ccycloud.quasar-peqjre.root.comops.site
11:40:59.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679900
11:40:59.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
11:40:59.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:59.584595 19781 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
11:40:59.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:59.585632 19781 file_cache.cc:493] Constructed file cache file cache with capacity 419430
11:40:59.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:59.602587 19788 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:41:00.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:59.602654 19791 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:41:00.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:40:59.602655 19789 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:41:00.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:40:59.604022 19781 server_base.cc:1034] running on GCE node
11:41:00.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:00.715348 19781 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
11:41:00.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:00.718046 19781 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
11:41:00.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:00.719334 19781 hybrid_clock.cc:648] HybridClock initialized: now 1790682060719299 us; error 37 us; skew 500 ppm
11:41:00.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:00.720144 19781 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
11:41:00.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 11:41:00 dist-test-slave-h1fj krb5kdc[18876](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790682060, etypes {rep=17 tkt=17 ses=17}, kudu/127.18.108.126@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
11:41:00.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:00.725502 19781 init.cc:377] Logged in from keytab as kudu/127.18.108.126@KRBTEST.COM (short username kudu)
11:41:00.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:00.730412 19781 webserver.cc:460] Webserver started at http://127.18.108.126:36579/ using document root <none> and password file <none>
11:41:00.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:00.731380 19781 fs_manager.cc:359] Metadata directory not provided
11:41:00.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:00.731600 19781 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
11:41:00.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:00.739166 19781 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.007s sys 0.000s
11:41:00.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:00.743278 19799 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
11:41:00.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:00.744226 19781 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.004s sys 0.000s
11:41:00.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:00.744524 19781 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-0/data,/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-0/wal
11:41:00.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "eb8f4e2d825e4e319fd704623a2750bf"
11:41:00.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 11:40:40 on dist-test-slave-h1fj"
11:41:00.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:00.746737 19781 fs_report.cc:383] FS layout report
11:41:00.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
11:41:00.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-0/wal
11:41:00.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-0/wal
11:41:00.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-0/data/data
11:41:00.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
11:41:00.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
11:41:00.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
11:41:00.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
11:41:00.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
11:41:00.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
11:41:00.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
11:41:00.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
11:41:00.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
11:41:00.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
11:41:00.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
11:41:00.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:00.788388 19781 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
11:41:00.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:00.793504 19781 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
11:41:00.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:00.793931 19781 kserver.cc:163] Server-wide thread pool size limit: 3276
11:41:00.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:00.861042 19781 rpc_server.cc:307] RPC server started. Bound to: 127.18.108.126:43211
11:41:00.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:00.861184 19850 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.18.108.126:43211 every 8 connection(s)
11:41:00.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:00.863734 19781 server_base.cc:1166] Dumped server information to /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-0/data/info.pb
11:41:00.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:00.865427 18865 external_mini_cluster.cc:1388] Started /tmp/dist-test-task4Z5jUl/build/tsan/bin/kudu as pid 19781
11:41:00.866 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
11:41:00.866 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.18.108.125:36939
11:41:00.866 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
11:41:00.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:00.867755 18865 external_mini_cluster.cc:1326] Running /tmp/dist-test-task4Z5jUl/build/tsan/bin/kudu
11:41:00.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task4Z5jUl/build/tsan/bin/kudu
11:41:00.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-1/wal
11:41:00.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-1/data
11:41:00.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
11:41:00.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
11:41:00.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
11:41:00.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:41:00.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:41:00.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:41:00.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-1/logs
11:41:00.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-1/data/info.pb
11:41:00.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:41:00.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
11:41:00.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
11:41:00.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
11:41:00.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
11:41:00.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:41:00.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:41:00.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:41:00.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
11:41:00.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
11:41:00.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
11:41:00.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
11:41:00.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.108.125:36939
11:41:00.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.108.125
11:41:00.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38867
11:41:00.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/krb5kdc/kudu_127.18.108.125.keytab
11:41:00.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.18.108.125
11:41:00.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
11:41:00.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
11:41:00.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
11:41:00.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.108.84:34171
11:41:00.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:41:00.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:41:00.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:41:00.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
11:41:00.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.18.108.126:43211,127.18.108.125:36939,127.18.108.124:42121
11:41:00.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
11:41:00.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45823/jwks.json
11:41:00.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
11:41:00.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:00.874548 19851 sys_catalog.cc:263] Verifying existing consensus state
11:41:00.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:00.879602 19851 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf: Bootstrap starting.
11:41:00.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:00.913363 19851 log.cc:824] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf: Log is configured to *not* fsync() on all Append() calls
11:41:00.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:00.929071 19851 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf: Bootstrap replayed 1/1 log segments. Stats: ops{read=6 overwritten=0 applied=6 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
11:41:00.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:00.929793 19851 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf: Bootstrap complete.
11:41:00.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:00.955070 19851 raft_consensus.cc:348] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [term 3 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eb8f4e2d825e4e319fd704623a2750bf" member_type: VOTER last_known_addr { host: "127.18.108.126" port: 43211 } } peers { permanent_uuid: "c716745b4a1b4b39957ab73274b4df06" member_type: VOTER last_known_addr { host: "127.18.108.125" port: 36939 } } peers { permanent_uuid: "5a5e3302510a40e28f4aa8f781a651ea" member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 } }
11:41:00.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:00.957280 19851 raft_consensus.cc:729] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [term 3 FOLLOWER]: Becoming Follower/Learner. State: Replica: eb8f4e2d825e4e319fd704623a2750bf, State: Initialized, Role: FOLLOWER
11:41:00.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:00.958094 19851 consensus_queue.cc:260] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 6, Last appended: 3.6, Last appended by leader: 6, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eb8f4e2d825e4e319fd704623a2750bf" member_type: VOTER last_known_addr { host: "127.18.108.126" port: 43211 } } peers { permanent_uuid: "c716745b4a1b4b39957ab73274b4df06" member_type: VOTER last_known_addr { host: "127.18.108.125" port: 36939 } } peers { permanent_uuid: "5a5e3302510a40e28f4aa8f781a651ea" member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 } }
11:41:00.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:00.960736 19855 sys_catalog.cc:455] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 3 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eb8f4e2d825e4e319fd704623a2750bf" member_type: VOTER last_known_addr { host: "127.18.108.126" port: 43211 } } peers { permanent_uuid: "c716745b4a1b4b39957ab73274b4df06" member_type: VOTER last_known_addr { host: "127.18.108.125" port: 36939 } } peers { permanent_uuid: "5a5e3302510a40e28f4aa8f781a651ea" member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 } } }
11:41:00.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:00.961545 19855 sys_catalog.cc:458] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [sys.catalog]: This master's current role is: FOLLOWER
11:41:00.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:00.962273 19851 sys_catalog.cc:564] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [sys.catalog]: configured and running, proceeding with master startup.
11:41:00.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:00.987500 19866 catalog_manager.cc:1259] Loaded cluster ID: 09fd2865e33b46ce9a354093f255891e
11:41:00.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:00.987738 19866 catalog_manager.cc:1552] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf: loading cluster ID for follower catalog manager: success
11:41:00.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:00.995605 19866 catalog_manager.cc:1574] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf: acquiring CA information for follower catalog manager: success
11:41:01.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:01.000998 19866 catalog_manager.cc:1602] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
11:41:01.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:01.024987 19781 master_runner.cc:186] Error getting master registration for 127.18.108.125:36939: Network error: Client connection negotiation failed: client connection to 127.18.108.125:36939: connect: Connection refused (error 111),
11:41:01.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:01.053802 19781 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
11:41:01.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:41:01.179665 19853 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
11:41:01.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:41:01.180233 19853 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
11:41:01.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:41:01.180892 19853 flags.cc:425] Enabled unsafe flag: --never_fsync=true
11:41:01.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:41:01.211426 19853 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
11:41:01.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:41:01.211666 19853 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
11:41:01.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:41:01.212029 19853 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
11:41:01.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:41:01.212289 19853 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
11:41:01.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:41:01.212491 19853 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
11:41:01.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:41:01.212723 19853 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
11:41:01.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:41:01.212867 19853 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:45823/jwks.json
11:41:01.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:01.248809 19853 master_runner.cc:386] Master server non-default flags:
11:41:01.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:41:01.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.108.84:34171
11:41:01.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:41:01.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:41:01.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-1/data
11:41:01.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-1/wal
11:41:01.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
11:41:01.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
11:41:01.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.18.108.126:43211,127.18.108.125:36939,127.18.108.124:42121
11:41:01.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/krb5kdc/kudu_127.18.108.125.keytab
11:41:01.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.18.108.125
11:41:01.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:41:01.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:41:01.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
11:41:01.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.108.125:36939
11:41:01.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
11:41:01.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
11:41:01.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
11:41:01.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45823/jwks.json
11:41:01.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
11:41:01.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
11:41:01.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
11:41:01.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:41:01.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:41:01.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-1/data/info.pb
11:41:01.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.108.125
11:41:01.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38867
11:41:01.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
11:41:01.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:41:01.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
11:41:01.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
11:41:01.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:41:01.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-1/logs
11:41:01.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:41:01.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
11:41:01.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
11:41:01.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
11:41:01.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
11:41:01.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
11:41:01.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
11:41:01.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 10:52:50 UTC on ccycloud.quasar-peqjre.root.comops.site
11:41:01.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679900
11:41:01.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
11:41:01.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:01.249847 19853 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
11:41:01.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:01.250798 19853 file_cache.cc:493] Constructed file cache file cache with capacity 419430
11:41:01.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:41:01.272008 19876 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:41:02.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:41:01.272169 19878 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:41:02.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:01.273808 19853 server_base.cc:1034] running on GCE node
11:41:02.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:41:01.273377 19875 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:41:02.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:02.441964 19853 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
11:41:02.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:02.447149 19853 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
11:41:02.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:02.448603 19853 hybrid_clock.cc:648] HybridClock initialized: now 1790682062448548 us; error 44 us; skew 500 ppm
11:41:02.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:02.449556 19853 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
11:41:02.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 11:41:02 dist-test-slave-h1fj krb5kdc[18876](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790682062, etypes {rep=17 tkt=17 ses=17}, kudu/127.18.108.125@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
11:41:02.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:02.459867 19853 init.cc:377] Logged in from keytab as kudu/127.18.108.125@KRBTEST.COM (short username kudu)
11:41:02.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:02.478854 19853 webserver.cc:460] Webserver started at http://127.18.108.125:38867/ using document root <none> and password file <none>
11:41:02.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:02.480093 19853 fs_manager.cc:359] Metadata directory not provided
11:41:02.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:02.480367 19853 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
11:41:02.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:02.491804 19853 fs_manager.cc:709] Time spent opening directory manager: real 0.006s user 0.001s sys 0.004s
11:41:02.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:02.497671 19887 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
11:41:02.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:02.499012 19853 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.005s sys 0.000s
11:41:02.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:02.499511 19853 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-1/data,/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-1/wal
11:41:02.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c716745b4a1b4b39957ab73274b4df06"
11:41:02.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 11:40:42 on dist-test-slave-h1fj"
11:41:02.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:02.505990 19853 fs_report.cc:383] FS layout report
11:41:02.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
11:41:02.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-1/wal
11:41:02.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-1/wal
11:41:02.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-1/data/data
11:41:02.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
11:41:02.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
11:41:02.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
11:41:02.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
11:41:02.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
11:41:02.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
11:41:02.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
11:41:02.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
11:41:02.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
11:41:02.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
11:41:02.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
11:41:02.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:02.547333 19853 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
11:41:02.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:02.554452 19853 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
11:41:02.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:02.554973 19853 kserver.cc:163] Server-wide thread pool size limit: 3276
11:41:02.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:02.624009 19924 raft_consensus.cc:482] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [term 3 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
11:41:02.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:02.624563 19924 raft_consensus.cc:504] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eb8f4e2d825e4e319fd704623a2750bf" member_type: VOTER last_known_addr { host: "127.18.108.126" port: 43211 } } peers { permanent_uuid: "c716745b4a1b4b39957ab73274b4df06" member_type: VOTER last_known_addr { host: "127.18.108.125" port: 36939 } } peers { permanent_uuid: "5a5e3302510a40e28f4aa8f781a651ea" member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 } }
11:41:02.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:02.638885 19924 leader_election.cc:288] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers c716745b4a1b4b39957ab73274b4df06 (127.18.108.125:36939), 5a5e3302510a40e28f4aa8f781a651ea (127.18.108.124:42121)
11:41:02.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:41:02.640971 19800 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.18.108.125:36939: connect: Connection refused (error 111)
11:41:02.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:41:02.646149 19802 leader_election.cc:334] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 5a5e3302510a40e28f4aa8f781a651ea (127.18.108.124:42121): Network error: Client connection negotiation failed: client connection to 127.18.108.124:42121: connect: Connection refused (error 111)
11:41:02.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:41:02.646636 19800 leader_election.cc:334] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer c716745b4a1b4b39957ab73274b4df06 (127.18.108.125:36939): Network error: Client connection negotiation failed: client connection to 127.18.108.125:36939: connect: Connection refused (error 111)
11:41:02.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:02.647017 19800 leader_election.cc:302] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: eb8f4e2d825e4e319fd704623a2750bf; no voters: 5a5e3302510a40e28f4aa8f781a651ea, c716745b4a1b4b39957ab73274b4df06
11:41:02.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:02.647779 19853 rpc_server.cc:307] RPC server started. Bound to: 127.18.108.125:36939
11:41:02.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:02.647914 19924 raft_consensus.cc:2726] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
11:41:02.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:02.647928 19942 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.18.108.125:36939 every 8 connection(s)
11:41:02.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:02.650621 19853 server_base.cc:1166] Dumped server information to /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-1/data/info.pb
11:41:02.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:02.658126 18865 external_mini_cluster.cc:1388] Started /tmp/dist-test-task4Z5jUl/build/tsan/bin/kudu as pid 19853
11:41:02.658 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
11:41:02.658 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.18.108.124:42121
11:41:02.658 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
11:41:02.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:02.660140 18865 external_mini_cluster.cc:1326] Running /tmp/dist-test-task4Z5jUl/build/tsan/bin/kudu
11:41:02.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task4Z5jUl/build/tsan/bin/kudu
11:41:02.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-2/wal
11:41:02.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-2/data
11:41:02.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
11:41:02.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
11:41:02.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
11:41:02.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:41:02.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:41:02.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:41:02.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-2/logs
11:41:02.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-2/data/info.pb
11:41:02.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:41:02.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
11:41:02.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
11:41:02.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
11:41:02.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
11:41:02.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:41:02.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:41:02.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:41:02.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
11:41:02.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
11:41:02.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
11:41:02.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
11:41:02.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.108.124:42121
11:41:02.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.108.124
11:41:02.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37519
11:41:02.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/krb5kdc/kudu_127.18.108.124.keytab
11:41:02.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.18.108.124
11:41:02.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
11:41:02.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
11:41:02.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
11:41:02.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.108.84:34171
11:41:02.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:41:02.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:41:02.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:41:02.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
11:41:02.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.18.108.126:43211,127.18.108.125:36939,127.18.108.124:42121
11:41:02.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
11:41:02.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45823/jwks.json
11:41:02.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
11:41:02.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:02.663816 19943 sys_catalog.cc:263] Verifying existing consensus state
11:41:02.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:02.671577 19943 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06: Bootstrap starting.
11:41:02.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:02.720065 19943 log.cc:824] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06: Log is configured to *not* fsync() on all Append() calls
11:41:02.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:02.740990 19943 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06: Bootstrap replayed 1/1 log segments. Stats: ops{read=6 overwritten=0 applied=6 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
11:41:02.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:02.741945 19943 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06: Bootstrap complete.
11:41:02.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:02.776523 19943 raft_consensus.cc:348] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [term 3 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eb8f4e2d825e4e319fd704623a2750bf" member_type: VOTER last_known_addr { host: "127.18.108.126" port: 43211 } } peers { permanent_uuid: "c716745b4a1b4b39957ab73274b4df06" member_type: VOTER last_known_addr { host: "127.18.108.125" port: 36939 } } peers { permanent_uuid: "5a5e3302510a40e28f4aa8f781a651ea" member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 } }
11:41:02.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:02.779801 19943 raft_consensus.cc:729] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [term 3 FOLLOWER]: Becoming Follower/Learner. State: Replica: c716745b4a1b4b39957ab73274b4df06, State: Initialized, Role: FOLLOWER
11:41:02.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:02.780689 19943 consensus_queue.cc:260] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 6, Last appended: 3.6, Last appended by leader: 6, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eb8f4e2d825e4e319fd704623a2750bf" member_type: VOTER last_known_addr { host: "127.18.108.126" port: 43211 } } peers { permanent_uuid: "c716745b4a1b4b39957ab73274b4df06" member_type: VOTER last_known_addr { host: "127.18.108.125" port: 36939 } } peers { permanent_uuid: "5a5e3302510a40e28f4aa8f781a651ea" member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 } }
11:41:02.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:02.783475 19947 sys_catalog.cc:455] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 3 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eb8f4e2d825e4e319fd704623a2750bf" member_type: VOTER last_known_addr { host: "127.18.108.126" port: 43211 } } peers { permanent_uuid: "c716745b4a1b4b39957ab73274b4df06" member_type: VOTER last_known_addr { host: "127.18.108.125" port: 36939 } } peers { permanent_uuid: "5a5e3302510a40e28f4aa8f781a651ea" member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 } } }
11:41:02.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:02.784670 19947 sys_catalog.cc:458] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [sys.catalog]: This master's current role is: FOLLOWER
11:41:02.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:02.785108 19943 sys_catalog.cc:564] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [sys.catalog]: configured and running, proceeding with master startup.
11:41:02.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:02.818696 19958 catalog_manager.cc:1259] Loaded cluster ID: 09fd2865e33b46ce9a354093f255891e
11:41:02.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:02.818956 19958 catalog_manager.cc:1552] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06: loading cluster ID for follower catalog manager: success
11:41:02.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:02.826256 19958 catalog_manager.cc:1574] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06: acquiring CA information for follower catalog manager: success
11:41:02.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 11:41:02 dist-test-slave-h1fj krb5kdc[18876](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790682062, etypes {rep=17 tkt=17 ses=17}, kudu/127.18.108.125@KRBTEST.COM for kudu/127.18.108.126@KRBTEST.COM
11:41:02.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:02.860832 19958 catalog_manager.cc:1602] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
11:41:02.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:02.902650 19853 master_runner.cc:186] Error getting master registration for 127.18.108.124:42121: Network error: Client connection negotiation failed: client connection to 127.18.108.124:42121: connect: Connection refused (error 111),
11:41:02.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:02.932739 19853 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
11:41:03.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:41:03.072491 19945 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
11:41:03.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:41:03.073051 19945 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
11:41:03.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:41:03.074011 19945 flags.cc:425] Enabled unsafe flag: --never_fsync=true
11:41:03.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:41:03.114981 19945 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
11:41:03.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:41:03.115302 19945 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
11:41:03.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:41:03.115736 19945 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
11:41:03.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:41:03.116037 19945 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
11:41:03.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:41:03.116338 19945 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
11:41:03.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:41:03.116629 19945 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
11:41:03.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:41:03.116842 19945 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:45823/jwks.json
11:41:03.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:03.161746 19945 master_runner.cc:386] Master server non-default flags:
11:41:03.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:41:03.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.108.84:34171
11:41:03.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:41:03.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:41:03.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-2/data
11:41:03.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-2/wal
11:41:03.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
11:41:03.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
11:41:03.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.18.108.126:43211,127.18.108.125:36939,127.18.108.124:42121
11:41:03.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/krb5kdc/kudu_127.18.108.124.keytab
11:41:03.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.18.108.124
11:41:03.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:41:03.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:41:03.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
11:41:03.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.108.124:42121
11:41:03.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
11:41:03.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
11:41:03.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
11:41:03.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45823/jwks.json
11:41:03.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
11:41:03.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
11:41:03.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
11:41:03.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:41:03.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:41:03.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-2/data/info.pb
11:41:03.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.108.124
11:41:03.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37519
11:41:03.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
11:41:03.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:41:03.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
11:41:03.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
11:41:03.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:41:03.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-2/logs
11:41:03.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:41:03.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
11:41:03.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
11:41:03.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
11:41:03.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
11:41:03.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
11:41:03.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
11:41:03.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 10:52:50 UTC on ccycloud.quasar-peqjre.root.comops.site
11:41:03.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679900
11:41:03.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
11:41:03.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:03.162792 19945 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
11:41:03.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:03.163981 19945 file_cache.cc:493] Constructed file cache file cache with capacity 419430
11:41:03.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:41:03.182801 19970 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:41:04.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:41:03.186997 19972 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:41:04.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:03.191299 19945 server_base.cc:1034] running on GCE node
11:41:04.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:41:03.185086 19969 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:41:04.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:04.348130 19945 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
11:41:04.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:04.351253 19945 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
11:41:04.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:04.352658 19945 hybrid_clock.cc:648] HybridClock initialized: now 1790682064352614 us; error 52 us; skew 500 ppm
11:41:04.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:04.353482 19945 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
11:41:04.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 11:41:04 dist-test-slave-h1fj krb5kdc[18876](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790682064, etypes {rep=17 tkt=17 ses=17}, kudu/127.18.108.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
11:41:04.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:04.359885 19945 init.cc:377] Logged in from keytab as kudu/127.18.108.124@KRBTEST.COM (short username kudu)
11:41:04.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:04.367890 19945 webserver.cc:460] Webserver started at http://127.18.108.124:37519/ using document root <none> and password file <none>
11:41:04.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:04.368830 19945 fs_manager.cc:359] Metadata directory not provided
11:41:04.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:04.369030 19945 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
11:41:04.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:04.377050 19945 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.001s sys 0.004s
11:41:04.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:04.381677 19980 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
11:41:04.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:04.382664 19945 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
11:41:04.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:04.382925 19945 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-2/data,/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-2/wal
11:41:04.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5a5e3302510a40e28f4aa8f781a651ea"
11:41:04.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 11:40:44 on dist-test-slave-h1fj"
11:41:04.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:04.385480 19945 fs_report.cc:383] FS layout report
11:41:04.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
11:41:04.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-2/wal
11:41:04.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-2/wal
11:41:04.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-2/data/data
11:41:04.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
11:41:04.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
11:41:04.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
11:41:04.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
11:41:04.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
11:41:04.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
11:41:04.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
11:41:04.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
11:41:04.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
11:41:04.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
11:41:04.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
11:41:04.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:04.418999 19945 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
11:41:04.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:04.424064 19985 raft_consensus.cc:482] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [term 3 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
11:41:04.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:04.424963 19945 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
11:41:04.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:04.424580 19985 raft_consensus.cc:504] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eb8f4e2d825e4e319fd704623a2750bf" member_type: VOTER last_known_addr { host: "127.18.108.126" port: 43211 } } peers { permanent_uuid: "c716745b4a1b4b39957ab73274b4df06" member_type: VOTER last_known_addr { host: "127.18.108.125" port: 36939 } } peers { permanent_uuid: "5a5e3302510a40e28f4aa8f781a651ea" member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 } }
11:41:04.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:04.425398 19945 kserver.cc:163] Server-wide thread pool size limit: 3276
11:41:04.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:04.428861 19826 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "c716745b4a1b4b39957ab73274b4df06" candidate_term: 4 candidate_status { last_received { term: 3 index: 6 } } ignore_live_leader: false dest_uuid: "eb8f4e2d825e4e319fd704623a2750bf" is_pre_election: true
11:41:04.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:04.429579 19826 raft_consensus.cc:2454] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [term 3 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c716745b4a1b4b39957ab73274b4df06 in term 3.
11:41:04.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:04.431025 19888 leader_election.cc:302] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: c716745b4a1b4b39957ab73274b4df06, eb8f4e2d825e4e319fd704623a2750bf; no voters:
11:41:04.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:04.435639 19985 leader_election.cc:288] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers eb8f4e2d825e4e319fd704623a2750bf (127.18.108.126:43211), 5a5e3302510a40e28f4aa8f781a651ea (127.18.108.124:42121)
11:41:04.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:04.436532 19985 raft_consensus.cc:2781] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [term 3 FOLLOWER]: Leader pre-election won for term 4
11:41:04.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:04.437335 19985 raft_consensus.cc:482] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [term 3 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
11:41:04.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:04.438051 19985 raft_consensus.cc:3037] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [term 3 FOLLOWER]: Advancing to term 4
11:41:04.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:41:04.444989 19890 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.18.108.124:42121: connect: Connection refused (error 111)
11:41:04.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:04.451500 19985 raft_consensus.cc:504] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [term 4 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eb8f4e2d825e4e319fd704623a2750bf" member_type: VOTER last_known_addr { host: "127.18.108.126" port: 43211 } } peers { permanent_uuid: "c716745b4a1b4b39957ab73274b4df06" member_type: VOTER last_known_addr { host: "127.18.108.125" port: 36939 } } peers { permanent_uuid: "5a5e3302510a40e28f4aa8f781a651ea" member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 } }
11:41:04.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:04.455130 19826 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "c716745b4a1b4b39957ab73274b4df06" candidate_term: 4 candidate_status { last_received { term: 3 index: 6 } } ignore_live_leader: false dest_uuid: "eb8f4e2d825e4e319fd704623a2750bf"
11:41:04.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:41:04.455559 19890 leader_election.cc:334] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 5a5e3302510a40e28f4aa8f781a651ea (127.18.108.124:42121): Network error: Client connection negotiation failed: client connection to 127.18.108.124:42121: connect: Connection refused (error 111)
11:41:04.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:04.455698 19826 raft_consensus.cc:3037] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [term 3 FOLLOWER]: Advancing to term 4
11:41:04.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:04.456818 19985 leader_election.cc:288] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [CANDIDATE]: Term 4 election: Requested vote from peers eb8f4e2d825e4e319fd704623a2750bf (127.18.108.126:43211), 5a5e3302510a40e28f4aa8f781a651ea (127.18.108.124:42121)
11:41:04.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:41:04.460713 19890 leader_election.cc:334] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [CANDIDATE]: Term 4 election: RPC error from VoteRequest() call to peer 5a5e3302510a40e28f4aa8f781a651ea (127.18.108.124:42121): Network error: Client connection negotiation failed: client connection to 127.18.108.124:42121: connect: Connection refused (error 111)
11:41:04.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:04.465583 19826 raft_consensus.cc:2454] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [term 4 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c716745b4a1b4b39957ab73274b4df06 in term 4.
11:41:04.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:04.466576 19888 leader_election.cc:302] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [CANDIDATE]: Term 4 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: c716745b4a1b4b39957ab73274b4df06, eb8f4e2d825e4e319fd704623a2750bf; no voters: 5a5e3302510a40e28f4aa8f781a651ea
11:41:04.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:04.467733 19985 raft_consensus.cc:2781] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [term 4 FOLLOWER]: Leader election won for term 4
11:41:04.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:04.468400 19985 raft_consensus.cc:686] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [term 4 LEADER]: Becoming Leader. State: Replica: c716745b4a1b4b39957ab73274b4df06, State: Running, Role: LEADER
11:41:04.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:04.469614 19985 consensus_queue.cc:237] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 3.6, Last appended by leader: 6, Current term: 4, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eb8f4e2d825e4e319fd704623a2750bf" member_type: VOTER last_known_addr { host: "127.18.108.126" port: 43211 } } peers { permanent_uuid: "c716745b4a1b4b39957ab73274b4df06" member_type: VOTER last_known_addr { host: "127.18.108.125" port: 36939 } } peers { permanent_uuid: "5a5e3302510a40e28f4aa8f781a651ea" member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 } }
11:41:04.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:04.486037 19988 sys_catalog.cc:455] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [sys.catalog]: SysCatalogTable state changed. Reason: New leader c716745b4a1b4b39957ab73274b4df06. Latest consensus state: current_term: 4 leader_uuid: "c716745b4a1b4b39957ab73274b4df06" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eb8f4e2d825e4e319fd704623a2750bf" member_type: VOTER last_known_addr { host: "127.18.108.126" port: 43211 } } peers { permanent_uuid: "c716745b4a1b4b39957ab73274b4df06" member_type: VOTER last_known_addr { host: "127.18.108.125" port: 36939 } } peers { permanent_uuid: "5a5e3302510a40e28f4aa8f781a651ea" member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 } } }
11:41:04.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:04.486845 19988 sys_catalog.cc:458] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [sys.catalog]: This master's current role is: LEADER
11:41:04.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:04.496376 19993 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
11:41:04.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:04.505230 19993 catalog_manager.cc:1484] Initializing Kudu cluster ID...
11:41:04.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:04.509665 19993 catalog_manager.cc:1259] Loaded cluster ID: 09fd2865e33b46ce9a354093f255891e
11:41:04.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:04.509943 19993 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
11:41:04.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:04.515022 19993 catalog_manager.cc:1504] Loading token signing keys...
11:41:04.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:04.519160 19993 catalog_manager.cc:5879] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06: Loaded TSK: 0
11:41:04.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:04.520691 19993 catalog_manager.cc:1514] Initializing in-progress tserver states...
11:41:04.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:04.562180 19945 rpc_server.cc:307] RPC server started. Bound to: 127.18.108.124:42121
11:41:04.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:04.562386 20037 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.18.108.124:42121 every 8 connection(s)
11:41:04.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:04.565519 19945 server_base.cc:1166] Dumped server information to /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-2/data/info.pb
11:41:04.568 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
11:41:04.568 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.18.108.125:36939(127.18.108.125:36939)] connecting to peer
11:41:04.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:04.567281 18865 external_mini_cluster.cc:1388] Started /tmp/dist-test-task4Z5jUl/build/tsan/bin/kudu as pid 19945
11:41:04.573 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.18.108.125:36939(127.18.108.125:36939)] Successfully connected to peer
11:41:04.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:04.581370 20038 sys_catalog.cc:263] Verifying existing consensus state
11:41:04.582 [DEBUG - kudu-nio-0] (SecurityContext.java:225) Refreshing Kerberos credentials...
11:41:04.583 [DEBUG - kudu-nio-0] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/krb5kdc/krb5cc
11:41:04.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 11:41:04 dist-test-slave-h1fj krb5kdc[18876](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790682058, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
11:41:04.588 [DEBUG - kudu-nio-0] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
11:41:04.589 [INFO - kudu-nio-0] (SecurityContext.java:259) Successfully refreshed Kerberos credentials from ticket cache
11:41:04.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:04.589208 20038 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5a5e3302510a40e28f4aa8f781a651ea: Bootstrap starting.
11:41:04.590 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.18.108.125
11:41:04.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 11:41:04 dist-test-slave-h1fj krb5kdc[18876](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790682058, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.18.108.125@KRBTEST.COM
11:41:04.620 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xedf1f55a, L:/127.0.0.1:34304 - R:/127.18.108.125:36939] using SASL/GSSAPI
11:41:04.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:04.643412 20038 log.cc:824] T 00000000000000000000000000000000 P 5a5e3302510a40e28f4aa8f781a651ea: Log is configured to *not* fsync() on all Append() calls
11:41:04.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:04.664656 20038 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5a5e3302510a40e28f4aa8f781a651ea: Bootstrap replayed 1/1 log segments. Stats: ops{read=6 overwritten=0 applied=6 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
11:41:04.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:04.665616 20038 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5a5e3302510a40e28f4aa8f781a651ea: Bootstrap complete.
11:41:04.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:04.700624 20038 raft_consensus.cc:348] T 00000000000000000000000000000000 P 5a5e3302510a40e28f4aa8f781a651ea [term 3 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eb8f4e2d825e4e319fd704623a2750bf" member_type: VOTER last_known_addr { host: "127.18.108.126" port: 43211 } } peers { permanent_uuid: "c716745b4a1b4b39957ab73274b4df06" member_type: VOTER last_known_addr { host: "127.18.108.125" port: 36939 } } peers { permanent_uuid: "5a5e3302510a40e28f4aa8f781a651ea" member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 } }
11:41:04.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:04.704046 20038 raft_consensus.cc:729] T 00000000000000000000000000000000 P 5a5e3302510a40e28f4aa8f781a651ea [term 3 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5a5e3302510a40e28f4aa8f781a651ea, State: Initialized, Role: FOLLOWER
11:41:04.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:04.704973 20038 consensus_queue.cc:260] T 00000000000000000000000000000000 P 5a5e3302510a40e28f4aa8f781a651ea [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 6, Last appended: 3.6, Last appended by leader: 6, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eb8f4e2d825e4e319fd704623a2750bf" member_type: VOTER last_known_addr { host: "127.18.108.126" port: 43211 } } peers { permanent_uuid: "c716745b4a1b4b39957ab73274b4df06" member_type: VOTER last_known_addr { host: "127.18.108.125" port: 36939 } } peers { permanent_uuid: "5a5e3302510a40e28f4aa8f781a651ea" member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 } }
11:41:04.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:04.707719 20042 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5a5e3302510a40e28f4aa8f781a651ea [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 3 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eb8f4e2d825e4e319fd704623a2750bf" member_type: VOTER last_known_addr { host: "127.18.108.126" port: 43211 } } peers { permanent_uuid: "c716745b4a1b4b39957ab73274b4df06" member_type: VOTER last_known_addr { host: "127.18.108.125" port: 36939 } } peers { permanent_uuid: "5a5e3302510a40e28f4aa8f781a651ea" member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 } } }
11:41:04.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:04.708366 20042 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5a5e3302510a40e28f4aa8f781a651ea [sys.catalog]: This master's current role is: FOLLOWER
11:41:04.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:04.709508 20038 sys_catalog.cc:564] T 00000000000000000000000000000000 P 5a5e3302510a40e28f4aa8f781a651ea [sys.catalog]: configured and running, proceeding with master startup.
11:41:04.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:04.747516 20053 catalog_manager.cc:1259] Loaded cluster ID: 09fd2865e33b46ce9a354093f255891e
11:41:04.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:04.747746 20053 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 5a5e3302510a40e28f4aa8f781a651ea: loading cluster ID for follower catalog manager: success
11:41:04.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 11:41:04 dist-test-slave-h1fj krb5kdc[18876](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790682064, etypes {rep=17 tkt=17 ses=17}, kudu/127.18.108.124@KRBTEST.COM for kudu/127.18.108.126@KRBTEST.COM
11:41:04.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:04.755431 20053 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 5a5e3302510a40e28f4aa8f781a651ea: acquiring CA information for follower catalog manager: success
11:41:04.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:04.760667 20053 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 5a5e3302510a40e28f4aa8f781a651ea: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
11:41:04.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:04.905854 20013 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 5a5e3302510a40e28f4aa8f781a651ea [term 3 FOLLOWER]: Advancing to term 4
11:41:04.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:04.908160 19826 raft_consensus.cc:1261] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [term 4 FOLLOWER]: Refusing update from remote peer c716745b4a1b4b39957ab73274b4df06: Log matching property violated. Preceding OpId in replica: term: 3 index: 6. Preceding OpId from leader: term: 4 index: 7. (index mismatch)
11:41:04.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:04.909852 19988 consensus_queue.cc:1035] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [LEADER]: Connected to new peer: Peer: permanent_uuid: "eb8f4e2d825e4e319fd704623a2750bf" member_type: VOTER last_known_addr { host: "127.18.108.126" port: 43211 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
11:41:04.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:04.914335 20013 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 5a5e3302510a40e28f4aa8f781a651ea [term 4 FOLLOWER]: Refusing update from remote peer c716745b4a1b4b39957ab73274b4df06: Log matching property violated. Preceding OpId in replica: term: 3 index: 6. Preceding OpId from leader: term: 4 index: 7. (index mismatch)
11:41:04.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:04.915750 19988 consensus_queue.cc:1035] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5a5e3302510a40e28f4aa8f781a651ea" member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
11:41:04.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:04.920678 20059 sys_catalog.cc:455] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [sys.catalog]: SysCatalogTable state changed. Reason: New leader c716745b4a1b4b39957ab73274b4df06. Latest consensus state: current_term: 4 leader_uuid: "c716745b4a1b4b39957ab73274b4df06" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eb8f4e2d825e4e319fd704623a2750bf" member_type: VOTER last_known_addr { host: "127.18.108.126" port: 43211 } } peers { permanent_uuid: "c716745b4a1b4b39957ab73274b4df06" member_type: VOTER last_known_addr { host: "127.18.108.125" port: 36939 } } peers { permanent_uuid: "5a5e3302510a40e28f4aa8f781a651ea" member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 } } }
11:41:04.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:04.921562 20059 sys_catalog.cc:458] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [sys.catalog]: This master's current role is: FOLLOWER
11:41:04.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:04.925422 20059 sys_catalog.cc:455] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 4 leader_uuid: "c716745b4a1b4b39957ab73274b4df06" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eb8f4e2d825e4e319fd704623a2750bf" member_type: VOTER last_known_addr { host: "127.18.108.126" port: 43211 } } peers { permanent_uuid: "c716745b4a1b4b39957ab73274b4df06" member_type: VOTER last_known_addr { host: "127.18.108.125" port: 36939 } } peers { permanent_uuid: "5a5e3302510a40e28f4aa8f781a651ea" member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 } } }
11:41:04.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:04.926088 20059 sys_catalog.cc:458] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [sys.catalog]: This master's current role is: FOLLOWER
11:41:04.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:04.927461 19988 sys_catalog.cc:455] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 4 leader_uuid: "c716745b4a1b4b39957ab73274b4df06" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eb8f4e2d825e4e319fd704623a2750bf" member_type: VOTER last_known_addr { host: "127.18.108.126" port: 43211 } } peers { permanent_uuid: "c716745b4a1b4b39957ab73274b4df06" member_type: VOTER last_known_addr { host: "127.18.108.125" port: 36939 } } peers { permanent_uuid: "5a5e3302510a40e28f4aa8f781a651ea" member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 } } }
11:41:04.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:04.928122 19988 sys_catalog.cc:458] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [sys.catalog]: This master's current role is: LEADER
11:41:04.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:04.937537 19988 sys_catalog.cc:455] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 4 leader_uuid: "c716745b4a1b4b39957ab73274b4df06" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eb8f4e2d825e4e319fd704623a2750bf" member_type: VOTER last_known_addr { host: "127.18.108.126" port: 43211 } } peers { permanent_uuid: "c716745b4a1b4b39957ab73274b4df06" member_type: VOTER last_known_addr { host: "127.18.108.125" port: 36939 } } peers { permanent_uuid: "5a5e3302510a40e28f4aa8f781a651ea" member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 } } }
11:41:04.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:04.938190 19988 sys_catalog.cc:458] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [sys.catalog]: This master's current role is: LEADER
11:41:04.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:04.941068 20042 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5a5e3302510a40e28f4aa8f781a651ea [sys.catalog]: SysCatalogTable state changed. Reason: New leader c716745b4a1b4b39957ab73274b4df06. Latest consensus state: current_term: 4 leader_uuid: "c716745b4a1b4b39957ab73274b4df06" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eb8f4e2d825e4e319fd704623a2750bf" member_type: VOTER last_known_addr { host: "127.18.108.126" port: 43211 } } peers { permanent_uuid: "c716745b4a1b4b39957ab73274b4df06" member_type: VOTER last_known_addr { host: "127.18.108.125" port: 36939 } } peers { permanent_uuid: "5a5e3302510a40e28f4aa8f781a651ea" member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 } } }
11:41:04.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:04.941972 20042 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5a5e3302510a40e28f4aa8f781a651ea [sys.catalog]: This master's current role is: FOLLOWER
11:41:04.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:04.942727 20042 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5a5e3302510a40e28f4aa8f781a651ea [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 4 leader_uuid: "c716745b4a1b4b39957ab73274b4df06" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eb8f4e2d825e4e319fd704623a2750bf" member_type: VOTER last_known_addr { host: "127.18.108.126" port: 43211 } } peers { permanent_uuid: "c716745b4a1b4b39957ab73274b4df06" member_type: VOTER last_known_addr { host: "127.18.108.125" port: 36939 } } peers { permanent_uuid: "5a5e3302510a40e28f4aa8f781a651ea" member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 } } }
11:41:04.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:04.943387 20042 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5a5e3302510a40e28f4aa8f781a651ea [sys.catalog]: This master's current role is: FOLLOWER
11:41:05.624 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.18.108.126:43211
11:41:05.624 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
11:41:05.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:05.624650 18865 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task4Z5jUl/build/tsan/bin/kudu with pid 19781
11:41:05.652 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
11:41:05.652 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.18.108.125:36939
11:41:05.652 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
11:41:05.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:05.652794 18865 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task4Z5jUl/build/tsan/bin/kudu with pid 19853
11:41:05.677 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.18.108.125:36939(127.18.108.125:36939)] handling channelInactive
11:41:05.677 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.18.108.125:36939(127.18.108.125:36939)] cleaning up while in state READY due to: connection closed
11:41:05.678 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
11:41:05.678 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.18.108.124:42121
11:41:05.678 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
11:41:05.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:05.678875 18865 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task4Z5jUl/build/tsan/bin/kudu with pid 19945
11:41:05.699 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
11:41:05.699 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.18.108.126:43211
11:41:05.699 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
11:41:05.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:05.701279 18865 external_mini_cluster.cc:1326] Running /tmp/dist-test-task4Z5jUl/build/tsan/bin/kudu
11:41:05.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task4Z5jUl/build/tsan/bin/kudu
11:41:05.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-0/wal
11:41:05.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-0/data
11:41:05.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
11:41:05.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
11:41:05.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
11:41:05.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:41:05.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:41:05.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:41:05.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-0/logs
11:41:05.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-0/data/info.pb
11:41:05.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:41:05.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
11:41:05.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
11:41:05.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
11:41:05.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
11:41:05.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:41:05.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:41:05.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:41:05.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
11:41:05.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
11:41:05.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
11:41:05.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
11:41:05.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.108.126:43211
11:41:05.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.108.126
11:41:05.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36579
11:41:05.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/krb5kdc/kudu_127.18.108.126.keytab
11:41:05.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.18.108.126
11:41:05.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
11:41:05.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
11:41:05.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
11:41:05.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.108.84:34171
11:41:05.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:41:05.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:41:05.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:41:05.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
11:41:05.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.18.108.126:43211,127.18.108.125:36939,127.18.108.124:42121
11:41:05.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
11:41:05.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45823/jwks.json
11:41:05.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
11:41:05.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:41:05.986778 20068 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
11:41:05.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:41:05.987337 20068 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
11:41:05.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:41:05.987982 20068 flags.cc:425] Enabled unsafe flag: --never_fsync=true
11:41:06.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:41:06.018447 20068 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
11:41:06.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:41:06.018644 20068 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
11:41:06.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:41:06.018901 20068 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
11:41:06.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:41:06.019093 20068 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
11:41:06.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:41:06.019340 20068 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
11:41:06.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:41:06.019543 20068 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
11:41:06.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:41:06.019682 20068 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:45823/jwks.json
11:41:06.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:06.054813 20068 master_runner.cc:386] Master server non-default flags:
11:41:06.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:41:06.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.108.84:34171
11:41:06.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:41:06.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:41:06.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-0/data
11:41:06.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-0/wal
11:41:06.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
11:41:06.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
11:41:06.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.18.108.126:43211,127.18.108.125:36939,127.18.108.124:42121
11:41:06.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/krb5kdc/kudu_127.18.108.126.keytab
11:41:06.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.18.108.126
11:41:06.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:41:06.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:41:06.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
11:41:06.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.108.126:43211
11:41:06.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
11:41:06.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
11:41:06.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
11:41:06.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45823/jwks.json
11:41:06.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
11:41:06.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
11:41:06.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
11:41:06.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:41:06.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:41:06.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-0/data/info.pb
11:41:06.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.108.126
11:41:06.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36579
11:41:06.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
11:41:06.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:41:06.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
11:41:06.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
11:41:06.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:41:06.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-0/logs
11:41:06.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:41:06.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
11:41:06.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
11:41:06.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
11:41:06.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
11:41:06.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
11:41:06.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
11:41:06.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 10:52:50 UTC on ccycloud.quasar-peqjre.root.comops.site
11:41:06.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679900
11:41:06.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
11:41:06.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:06.055905 20068 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
11:41:06.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:06.056845 20068 file_cache.cc:493] Constructed file cache file cache with capacity 419430
11:41:06.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:41:06.073216 20075 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:41:07.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:06.074874 20068 server_base.cc:1034] running on GCE node
11:41:07.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:41:06.075657 20078 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:41:07.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:41:06.073351 20076 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:41:07.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:07.192164 20068 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
11:41:07.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:07.194865 20068 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
11:41:07.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:07.196202 20068 hybrid_clock.cc:648] HybridClock initialized: now 1790682067196161 us; error 46 us; skew 500 ppm
11:41:07.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:07.197050 20068 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
11:41:07.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 11:41:07 dist-test-slave-h1fj krb5kdc[18876](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790682067, etypes {rep=17 tkt=17 ses=17}, kudu/127.18.108.126@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
11:41:07.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:07.202580 20068 init.cc:377] Logged in from keytab as kudu/127.18.108.126@KRBTEST.COM (short username kudu)
11:41:07.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:07.207420 20068 webserver.cc:460] Webserver started at http://127.18.108.126:36579/ using document root <none> and password file <none>
11:41:07.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:07.208329 20068 fs_manager.cc:359] Metadata directory not provided
11:41:07.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:07.208529 20068 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
11:41:07.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:07.216053 20068 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.003s sys 0.001s
11:41:07.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:07.220335 20088 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
11:41:07.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:07.221263 20068 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.004s sys 0.001s
11:41:07.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:07.221561 20068 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-0/data,/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-0/wal
11:41:07.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "eb8f4e2d825e4e319fd704623a2750bf"
11:41:07.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 11:40:40 on dist-test-slave-h1fj"
11:41:07.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:07.223798 20068 fs_report.cc:383] FS layout report
11:41:07.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
11:41:07.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-0/wal
11:41:07.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-0/wal
11:41:07.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-0/data/data
11:41:07.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
11:41:07.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
11:41:07.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
11:41:07.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
11:41:07.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
11:41:07.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
11:41:07.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
11:41:07.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
11:41:07.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
11:41:07.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
11:41:07.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
11:41:07.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:07.271631 20068 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
11:41:07.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:07.276612 20068 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
11:41:07.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:07.277070 20068 kserver.cc:163] Server-wide thread pool size limit: 3276
11:41:07.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:07.346853 20068 rpc_server.cc:307] RPC server started. Bound to: 127.18.108.126:43211
11:41:07.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:07.346983 20139 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.18.108.126:43211 every 8 connection(s)
11:41:07.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:07.349526 20068 server_base.cc:1166] Dumped server information to /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-0/data/info.pb
11:41:07.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:07.352366 18865 external_mini_cluster.cc:1388] Started /tmp/dist-test-task4Z5jUl/build/tsan/bin/kudu as pid 20068
11:41:07.353 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
11:41:07.353 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.18.108.125:36939
11:41:07.353 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
11:41:07.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:07.354640 18865 external_mini_cluster.cc:1326] Running /tmp/dist-test-task4Z5jUl/build/tsan/bin/kudu
11:41:07.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task4Z5jUl/build/tsan/bin/kudu
11:41:07.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-1/wal
11:41:07.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-1/data
11:41:07.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
11:41:07.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
11:41:07.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
11:41:07.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:41:07.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:41:07.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:41:07.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-1/logs
11:41:07.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-1/data/info.pb
11:41:07.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:41:07.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
11:41:07.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
11:41:07.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
11:41:07.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
11:41:07.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:41:07.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:41:07.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:41:07.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
11:41:07.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
11:41:07.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
11:41:07.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
11:41:07.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.108.125:36939
11:41:07.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.108.125
11:41:07.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38867
11:41:07.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/krb5kdc/kudu_127.18.108.125.keytab
11:41:07.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.18.108.125
11:41:07.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
11:41:07.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
11:41:07.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
11:41:07.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.108.84:34171
11:41:07.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:41:07.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:41:07.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:41:07.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
11:41:07.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.18.108.126:43211,127.18.108.125:36939,127.18.108.124:42121
11:41:07.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
11:41:07.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45823/jwks.json
11:41:07.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
11:41:07.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:07.360991 20140 sys_catalog.cc:263] Verifying existing consensus state
11:41:07.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:07.366279 20140 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf: Bootstrap starting.
11:41:07.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:07.401005 20140 log.cc:824] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf: Log is configured to *not* fsync() on all Append() calls
11:41:07.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:07.418447 20140 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf: Bootstrap replayed 1/1 log segments. Stats: ops{read=7 overwritten=0 applied=7 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
11:41:07.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:07.419130 20140 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf: Bootstrap complete.
11:41:07.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:07.441344 20140 raft_consensus.cc:348] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [term 4 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eb8f4e2d825e4e319fd704623a2750bf" member_type: VOTER last_known_addr { host: "127.18.108.126" port: 43211 } } peers { permanent_uuid: "c716745b4a1b4b39957ab73274b4df06" member_type: VOTER last_known_addr { host: "127.18.108.125" port: 36939 } } peers { permanent_uuid: "5a5e3302510a40e28f4aa8f781a651ea" member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 } }
11:41:07.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:07.443706 20140 raft_consensus.cc:729] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [term 4 FOLLOWER]: Becoming Follower/Learner. State: Replica: eb8f4e2d825e4e319fd704623a2750bf, State: Initialized, Role: FOLLOWER
11:41:07.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:07.444561 20140 consensus_queue.cc:260] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 7, Last appended: 4.7, Last appended by leader: 7, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eb8f4e2d825e4e319fd704623a2750bf" member_type: VOTER last_known_addr { host: "127.18.108.126" port: 43211 } } peers { permanent_uuid: "c716745b4a1b4b39957ab73274b4df06" member_type: VOTER last_known_addr { host: "127.18.108.125" port: 36939 } } peers { permanent_uuid: "5a5e3302510a40e28f4aa8f781a651ea" member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 } }
11:41:07.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:07.447115 20144 sys_catalog.cc:455] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 4 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eb8f4e2d825e4e319fd704623a2750bf" member_type: VOTER last_known_addr { host: "127.18.108.126" port: 43211 } } peers { permanent_uuid: "c716745b4a1b4b39957ab73274b4df06" member_type: VOTER last_known_addr { host: "127.18.108.125" port: 36939 } } peers { permanent_uuid: "5a5e3302510a40e28f4aa8f781a651ea" member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 } } }
11:41:07.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:07.447942 20144 sys_catalog.cc:458] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [sys.catalog]: This master's current role is: FOLLOWER
11:41:07.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:07.448655 20140 sys_catalog.cc:564] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [sys.catalog]: configured and running, proceeding with master startup.
11:41:07.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:07.485540 20155 catalog_manager.cc:1259] Loaded cluster ID: 09fd2865e33b46ce9a354093f255891e
11:41:07.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:07.485773 20155 catalog_manager.cc:1552] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf: loading cluster ID for follower catalog manager: success
11:41:07.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:07.493269 20155 catalog_manager.cc:1574] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf: acquiring CA information for follower catalog manager: success
11:41:07.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:07.498708 20155 catalog_manager.cc:1602] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
11:41:07.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 11:41:07 dist-test-slave-h1fj krb5kdc[18876](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790682067, etypes {rep=17 tkt=17 ses=17}, kudu/127.18.108.126@KRBTEST.COM for kudu/127.18.108.126@KRBTEST.COM
11:41:07.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:07.525744 20068 master_runner.cc:186] Error getting master registration for 127.18.108.125:36939: Network error: Client connection negotiation failed: client connection to 127.18.108.125:36939: connect: Connection refused (error 111),
11:41:07.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:07.557825 20068 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
11:41:07.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:41:07.682101 20142 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
11:41:07.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:41:07.682703 20142 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
11:41:07.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:41:07.683418 20142 flags.cc:425] Enabled unsafe flag: --never_fsync=true
11:41:07.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:41:07.714655 20142 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
11:41:07.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:41:07.714887 20142 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
11:41:07.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:41:07.715324 20142 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
11:41:07.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:41:07.715611 20142 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
11:41:07.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:41:07.715902 20142 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
11:41:07.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:41:07.716181 20142 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
11:41:07.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:41:07.716423 20142 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:45823/jwks.json
11:41:07.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:07.753851 20142 master_runner.cc:386] Master server non-default flags:
11:41:07.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:41:07.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.108.84:34171
11:41:07.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:41:07.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:41:07.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-1/data
11:41:07.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-1/wal
11:41:07.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
11:41:07.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
11:41:07.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.18.108.126:43211,127.18.108.125:36939,127.18.108.124:42121
11:41:07.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/krb5kdc/kudu_127.18.108.125.keytab
11:41:07.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.18.108.125
11:41:07.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:41:07.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:41:07.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
11:41:07.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.108.125:36939
11:41:07.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
11:41:07.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
11:41:07.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
11:41:07.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45823/jwks.json
11:41:07.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
11:41:07.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
11:41:07.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
11:41:07.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:41:07.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:41:07.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-1/data/info.pb
11:41:07.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.108.125
11:41:07.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38867
11:41:07.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
11:41:07.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:41:07.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
11:41:07.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
11:41:07.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:41:07.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-1/logs
11:41:07.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:41:07.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
11:41:07.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
11:41:07.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
11:41:07.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
11:41:07.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
11:41:07.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
11:41:07.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 10:52:50 UTC on ccycloud.quasar-peqjre.root.comops.site
11:41:07.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679900
11:41:07.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
11:41:07.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:07.755198 20142 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
11:41:07.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:07.756488 20142 file_cache.cc:493] Constructed file cache file cache with capacity 419430
11:41:07.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:41:07.774125 20165 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:41:08.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:08.896405 20172 raft_consensus.cc:482] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [term 4 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
11:41:08.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:08.896862 20172 raft_consensus.cc:504] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eb8f4e2d825e4e319fd704623a2750bf" member_type: VOTER last_known_addr { host: "127.18.108.126" port: 43211 } } peers { permanent_uuid: "c716745b4a1b4b39957ab73274b4df06" member_type: VOTER last_known_addr { host: "127.18.108.125" port: 36939 } } peers { permanent_uuid: "5a5e3302510a40e28f4aa8f781a651ea" member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 } }
11:41:08.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:08.900063 20172 leader_election.cc:288] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers c716745b4a1b4b39957ab73274b4df06 (127.18.108.125:36939), 5a5e3302510a40e28f4aa8f781a651ea (127.18.108.124:42121)
11:41:08.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:41:08.914497 20091 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.18.108.124:42121: connect: Connection refused (error 111)
11:41:08.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:41:08.919551 20089 leader_election.cc:334] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer c716745b4a1b4b39957ab73274b4df06 (127.18.108.125:36939): Network error: Client connection negotiation failed: client connection to 127.18.108.125:36939: connect: Connection refused (error 111)
11:41:08.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:41:08.920146 20091 leader_election.cc:334] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer 5a5e3302510a40e28f4aa8f781a651ea (127.18.108.124:42121): Network error: Client connection negotiation failed: client connection to 127.18.108.124:42121: connect: Connection refused (error 111)
11:41:08.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:08.920466 20091 leader_election.cc:302] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: eb8f4e2d825e4e319fd704623a2750bf; no voters: 5a5e3302510a40e28f4aa8f781a651ea, c716745b4a1b4b39957ab73274b4df06
11:41:08.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:08.921190 20172 raft_consensus.cc:2726] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [term 4 FOLLOWER]: Leader pre-election lost for term 5. Reason: could not achieve majority
11:41:08.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:41:07.775688 20164 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:41:08.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:41:07.775225 20167 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:41:08.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:07.778824 20142 server_base.cc:1034] running on GCE node
11:41:08.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:08.984798 20142 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
11:41:08.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:08.987463 20142 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
11:41:08.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:08.988837 20142 hybrid_clock.cc:648] HybridClock initialized: now 1790682068988811 us; error 37 us; skew 500 ppm
11:41:08.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:08.989655 20142 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
11:41:08.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 11:41:08 dist-test-slave-h1fj krb5kdc[18876](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790682068, etypes {rep=17 tkt=17 ses=17}, kudu/127.18.108.125@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
11:41:09.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:08.999864 20142 init.cc:377] Logged in from keytab as kudu/127.18.108.125@KRBTEST.COM (short username kudu)
11:41:09.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:09.005280 20142 webserver.cc:460] Webserver started at http://127.18.108.125:38867/ using document root <none> and password file <none>
11:41:09.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:09.006198 20142 fs_manager.cc:359] Metadata directory not provided
11:41:09.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:09.006397 20142 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
11:41:09.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:09.014824 20142 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.006s sys 0.000s
11:41:09.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:09.019526 20181 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
11:41:09.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:09.020593 20142 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.001s sys 0.003s
11:41:09.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:09.020910 20142 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-1/data,/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-1/wal
11:41:09.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c716745b4a1b4b39957ab73274b4df06"
11:41:09.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 11:40:42 on dist-test-slave-h1fj"
11:41:09.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:09.023027 20142 fs_report.cc:383] FS layout report
11:41:09.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
11:41:09.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-1/wal
11:41:09.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-1/wal
11:41:09.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-1/data/data
11:41:09.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
11:41:09.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
11:41:09.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
11:41:09.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
11:41:09.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
11:41:09.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
11:41:09.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
11:41:09.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
11:41:09.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
11:41:09.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
11:41:09.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
11:41:09.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:09.061333 20142 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
11:41:09.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:09.067549 20142 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
11:41:09.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:09.068080 20142 kserver.cc:163] Server-wide thread pool size limit: 3276
11:41:09.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:09.154649 20142 rpc_server.cc:307] RPC server started. Bound to: 127.18.108.125:36939
11:41:09.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:09.154767 20232 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.18.108.125:36939 every 8 connection(s)
11:41:09.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:09.158301 20142 server_base.cc:1166] Dumped server information to /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-1/data/info.pb
11:41:09.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:09.159616 18865 external_mini_cluster.cc:1388] Started /tmp/dist-test-task4Z5jUl/build/tsan/bin/kudu as pid 20142
11:41:09.160 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
11:41:09.160 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.18.108.124:42121
11:41:09.160 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
11:41:09.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:09.163542 18865 external_mini_cluster.cc:1326] Running /tmp/dist-test-task4Z5jUl/build/tsan/bin/kudu
11:41:09.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task4Z5jUl/build/tsan/bin/kudu
11:41:09.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-2/wal
11:41:09.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-2/data
11:41:09.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
11:41:09.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
11:41:09.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
11:41:09.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:41:09.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:41:09.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:41:09.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-2/logs
11:41:09.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-2/data/info.pb
11:41:09.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:41:09.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
11:41:09.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
11:41:09.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
11:41:09.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
11:41:09.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:41:09.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:41:09.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:41:09.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
11:41:09.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
11:41:09.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
11:41:09.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
11:41:09.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.108.124:42121
11:41:09.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.108.124
11:41:09.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37519
11:41:09.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/krb5kdc/kudu_127.18.108.124.keytab
11:41:09.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.18.108.124
11:41:09.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
11:41:09.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
11:41:09.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
11:41:09.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.108.84:34171
11:41:09.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:41:09.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:41:09.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:41:09.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
11:41:09.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.18.108.126:43211,127.18.108.125:36939,127.18.108.124:42121
11:41:09.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
11:41:09.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45823/jwks.json
11:41:09.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
11:41:09.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:09.175614 20233 sys_catalog.cc:263] Verifying existing consensus state
11:41:09.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:09.181951 20233 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06: Bootstrap starting.
11:41:09.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:09.217994 20233 log.cc:824] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06: Log is configured to *not* fsync() on all Append() calls
11:41:09.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:09.235781 20233 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06: Bootstrap replayed 1/1 log segments. Stats: ops{read=7 overwritten=0 applied=7 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
11:41:09.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:09.236519 20233 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06: Bootstrap complete.
11:41:09.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:09.260965 20233 raft_consensus.cc:348] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [term 4 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eb8f4e2d825e4e319fd704623a2750bf" member_type: VOTER last_known_addr { host: "127.18.108.126" port: 43211 } } peers { permanent_uuid: "c716745b4a1b4b39957ab73274b4df06" member_type: VOTER last_known_addr { host: "127.18.108.125" port: 36939 } } peers { permanent_uuid: "5a5e3302510a40e28f4aa8f781a651ea" member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 } }
11:41:09.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:09.263171 20233 raft_consensus.cc:729] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [term 4 FOLLOWER]: Becoming Follower/Learner. State: Replica: c716745b4a1b4b39957ab73274b4df06, State: Initialized, Role: FOLLOWER
11:41:09.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:09.264135 20233 consensus_queue.cc:260] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 7, Last appended: 4.7, Last appended by leader: 7, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eb8f4e2d825e4e319fd704623a2750bf" member_type: VOTER last_known_addr { host: "127.18.108.126" port: 43211 } } peers { permanent_uuid: "c716745b4a1b4b39957ab73274b4df06" member_type: VOTER last_known_addr { host: "127.18.108.125" port: 36939 } } peers { permanent_uuid: "5a5e3302510a40e28f4aa8f781a651ea" member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 } }
11:41:09.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:09.266887 20237 sys_catalog.cc:455] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 4 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eb8f4e2d825e4e319fd704623a2750bf" member_type: VOTER last_known_addr { host: "127.18.108.126" port: 43211 } } peers { permanent_uuid: "c716745b4a1b4b39957ab73274b4df06" member_type: VOTER last_known_addr { host: "127.18.108.125" port: 36939 } } peers { permanent_uuid: "5a5e3302510a40e28f4aa8f781a651ea" member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 } } }
11:41:09.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:09.268318 20237 sys_catalog.cc:458] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [sys.catalog]: This master's current role is: FOLLOWER
11:41:09.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:09.268555 20233 sys_catalog.cc:564] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [sys.catalog]: configured and running, proceeding with master startup.
11:41:09.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:09.304421 20248 catalog_manager.cc:795] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
11:41:09.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 11:41:09 dist-test-slave-h1fj krb5kdc[18876](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790682068, etypes {rep=17 tkt=17 ses=17}, kudu/127.18.108.125@KRBTEST.COM for kudu/127.18.108.126@KRBTEST.COM
11:41:09.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 11:41:09 dist-test-slave-h1fj krb5kdc[18876](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790682068, etypes {rep=17 tkt=17 ses=17}, kudu/127.18.108.125@KRBTEST.COM for kudu/127.18.108.125@KRBTEST.COM
11:41:09.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:09.399017 20142 master_runner.cc:186] Error getting master registration for 127.18.108.124:42121: Network error: Client connection negotiation failed: client connection to 127.18.108.124:42121: connect: Connection refused (error 111),
11:41:09.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:09.429473 20142 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
11:41:09.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:41:09.541085 20235 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
11:41:09.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:41:09.541615 20235 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
11:41:09.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:41:09.542245 20235 flags.cc:425] Enabled unsafe flag: --never_fsync=true
11:41:09.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:41:09.575073 20235 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
11:41:09.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:41:09.575344 20235 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
11:41:09.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:41:09.575639 20235 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
11:41:09.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:41:09.575831 20235 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
11:41:09.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:41:09.576040 20235 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
11:41:09.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:41:09.576246 20235 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
11:41:09.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:41:09.576385 20235 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:45823/jwks.json
11:41:09.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:09.613703 20235 master_runner.cc:386] Master server non-default flags:
11:41:09.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:41:09.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.108.84:34171
11:41:09.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:41:09.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:41:09.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-2/data
11:41:09.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-2/wal
11:41:09.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
11:41:09.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
11:41:09.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.18.108.126:43211,127.18.108.125:36939,127.18.108.124:42121
11:41:09.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/krb5kdc/kudu_127.18.108.124.keytab
11:41:09.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.18.108.124
11:41:09.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:41:09.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:41:09.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
11:41:09.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.108.124:42121
11:41:09.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
11:41:09.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
11:41:09.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
11:41:09.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45823/jwks.json
11:41:09.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
11:41:09.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
11:41:09.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
11:41:09.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:41:09.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:41:09.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-2/data/info.pb
11:41:09.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.108.124
11:41:09.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37519
11:41:09.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
11:41:09.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:41:09.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
11:41:09.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
11:41:09.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:41:09.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-2/logs
11:41:09.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:41:09.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
11:41:09.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
11:41:09.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
11:41:09.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
11:41:09.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
11:41:09.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
11:41:09.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 10:52:50 UTC on ccycloud.quasar-peqjre.root.comops.site
11:41:09.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679900
11:41:09.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
11:41:09.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:09.614770 20235 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
11:41:09.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:09.615779 20235 file_cache.cc:493] Constructed file cache file cache with capacity 419430
11:41:09.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:41:09.632282 20260 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:41:10.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:10.311691 20248 catalog_manager.cc:1259] Loaded cluster ID: 09fd2865e33b46ce9a354093f255891e
11:41:10.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:10.311903 20248 catalog_manager.cc:1552] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06: loading cluster ID for follower catalog manager: success
11:41:10.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:10.316987 20248 catalog_manager.cc:1574] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06: acquiring CA information for follower catalog manager: success
11:41:10.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:10.320732 20248 catalog_manager.cc:1602] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
11:41:10.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:10.623920 20266 raft_consensus.cc:482] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [term 4 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
11:41:10.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:10.624500 20266 raft_consensus.cc:504] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eb8f4e2d825e4e319fd704623a2750bf" member_type: VOTER last_known_addr { host: "127.18.108.126" port: 43211 } } peers { permanent_uuid: "c716745b4a1b4b39957ab73274b4df06" member_type: VOTER last_known_addr { host: "127.18.108.125" port: 36939 } } peers { permanent_uuid: "5a5e3302510a40e28f4aa8f781a651ea" member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 } }
11:41:10.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:10.627511 20266 leader_election.cc:288] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers eb8f4e2d825e4e319fd704623a2750bf (127.18.108.126:43211), 5a5e3302510a40e28f4aa8f781a651ea (127.18.108.124:42121)
11:41:10.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:10.628753 20115 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "c716745b4a1b4b39957ab73274b4df06" candidate_term: 5 candidate_status { last_received { term: 4 index: 7 } } ignore_live_leader: false dest_uuid: "eb8f4e2d825e4e319fd704623a2750bf" is_pre_election: true
11:41:10.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:10.629513 20115 raft_consensus.cc:2454] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [term 4 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c716745b4a1b4b39957ab73274b4df06 in term 4.
11:41:10.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:10.630766 20182 leader_election.cc:302] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: c716745b4a1b4b39957ab73274b4df06, eb8f4e2d825e4e319fd704623a2750bf; no voters:
11:41:10.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:10.632411 20266 raft_consensus.cc:2781] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [term 4 FOLLOWER]: Leader pre-election won for term 5
11:41:10.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:41:10.633594 20184 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.18.108.124:42121: connect: Connection refused (error 111)
11:41:10.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:10.633102 20266 raft_consensus.cc:482] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [term 4 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
11:41:10.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:10.634272 20266 raft_consensus.cc:3037] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [term 4 FOLLOWER]: Advancing to term 5
11:41:10.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:10.643272 20266 raft_consensus.cc:504] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [term 5 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eb8f4e2d825e4e319fd704623a2750bf" member_type: VOTER last_known_addr { host: "127.18.108.126" port: 43211 } } peers { permanent_uuid: "c716745b4a1b4b39957ab73274b4df06" member_type: VOTER last_known_addr { host: "127.18.108.125" port: 36939 } } peers { permanent_uuid: "5a5e3302510a40e28f4aa8f781a651ea" member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 } }
11:41:10.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:10.646052 20115 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "c716745b4a1b4b39957ab73274b4df06" candidate_term: 5 candidate_status { last_received { term: 4 index: 7 } } ignore_live_leader: false dest_uuid: "eb8f4e2d825e4e319fd704623a2750bf"
11:41:10.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:10.646571 20115 raft_consensus.cc:3037] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [term 4 FOLLOWER]: Advancing to term 5
11:41:10.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:41:10.648759 20184 leader_election.cc:334] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer 5a5e3302510a40e28f4aa8f781a651ea (127.18.108.124:42121): Network error: Client connection negotiation failed: client connection to 127.18.108.124:42121: connect: Connection refused (error 111)
11:41:10.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:10.649394 20266 leader_election.cc:288] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [CANDIDATE]: Term 5 election: Requested vote from peers eb8f4e2d825e4e319fd704623a2750bf (127.18.108.126:43211), 5a5e3302510a40e28f4aa8f781a651ea (127.18.108.124:42121)
11:41:10.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:41:10.654131 20184 leader_election.cc:334] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [CANDIDATE]: Term 5 election: RPC error from VoteRequest() call to peer 5a5e3302510a40e28f4aa8f781a651ea (127.18.108.124:42121): Network error: Client connection negotiation failed: client connection to 127.18.108.124:42121: connect: Connection refused (error 111)
11:41:10.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:10.655216 20115 raft_consensus.cc:2454] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [term 5 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c716745b4a1b4b39957ab73274b4df06 in term 5.
11:41:10.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:10.656122 20182 leader_election.cc:302] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [CANDIDATE]: Term 5 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: c716745b4a1b4b39957ab73274b4df06, eb8f4e2d825e4e319fd704623a2750bf; no voters: 5a5e3302510a40e28f4aa8f781a651ea
11:41:10.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:10.656836 20266 raft_consensus.cc:2781] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [term 5 FOLLOWER]: Leader election won for term 5
11:41:10.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:10.658730 20266 raft_consensus.cc:686] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [term 5 LEADER]: Becoming Leader. State: Replica: c716745b4a1b4b39957ab73274b4df06, State: Running, Role: LEADER
11:41:10.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:10.659598 20266 consensus_queue.cc:237] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 4.7, Last appended by leader: 7, Current term: 5, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eb8f4e2d825e4e319fd704623a2750bf" member_type: VOTER last_known_addr { host: "127.18.108.126" port: 43211 } } peers { permanent_uuid: "c716745b4a1b4b39957ab73274b4df06" member_type: VOTER last_known_addr { host: "127.18.108.125" port: 36939 } } peers { permanent_uuid: "5a5e3302510a40e28f4aa8f781a651ea" member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 } }
11:41:10.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:10.681167 20269 sys_catalog.cc:455] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [sys.catalog]: SysCatalogTable state changed. Reason: New leader c716745b4a1b4b39957ab73274b4df06. Latest consensus state: current_term: 5 leader_uuid: "c716745b4a1b4b39957ab73274b4df06" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eb8f4e2d825e4e319fd704623a2750bf" member_type: VOTER last_known_addr { host: "127.18.108.126" port: 43211 } } peers { permanent_uuid: "c716745b4a1b4b39957ab73274b4df06" member_type: VOTER last_known_addr { host: "127.18.108.125" port: 36939 } } peers { permanent_uuid: "5a5e3302510a40e28f4aa8f781a651ea" member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 } } }
11:41:10.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:10.681960 20269 sys_catalog.cc:458] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [sys.catalog]: This master's current role is: LEADER
11:41:10.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:10.684808 20271 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
11:41:10.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:10.693285 20271 catalog_manager.cc:1484] Initializing Kudu cluster ID...
11:41:10.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:10.697271 20271 catalog_manager.cc:1259] Loaded cluster ID: 09fd2865e33b46ce9a354093f255891e
11:41:10.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:10.697504 20271 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
11:41:10.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:10.702487 20271 catalog_manager.cc:1504] Loading token signing keys...
11:41:10.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:10.706516 20271 catalog_manager.cc:5879] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06: Loaded TSK: 0
11:41:10.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:10.707980 20271 catalog_manager.cc:1514] Initializing in-progress tserver states...
11:41:10.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:41:09.632335 20259 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:41:10.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:09.634002 20235 server_base.cc:1034] running on GCE node
11:41:10.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:41:09.632344 20262 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:41:10.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:10.796530 20235 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
11:41:10.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:10.799576 20235 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
11:41:10.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:10.800993 20235 hybrid_clock.cc:648] HybridClock initialized: now 1790682070800942 us; error 36 us; skew 500 ppm
11:41:10.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:10.802049 20235 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
11:41:10.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 11:41:10 dist-test-slave-h1fj krb5kdc[18876](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790682070, etypes {rep=17 tkt=17 ses=17}, kudu/127.18.108.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
11:41:10.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:10.808897 20235 init.cc:377] Logged in from keytab as kudu/127.18.108.124@KRBTEST.COM (short username kudu)
11:41:10.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:10.814248 20235 webserver.cc:460] Webserver started at http://127.18.108.124:37519/ using document root <none> and password file <none>
11:41:10.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:10.815444 20235 fs_manager.cc:359] Metadata directory not provided
11:41:10.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:10.815836 20235 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
11:41:10.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:10.826005 20235 fs_manager.cc:709] Time spent opening directory manager: real 0.006s user 0.005s sys 0.000s
11:41:10.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:10.831351 20277 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
11:41:10.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:10.832602 20235 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
11:41:10.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:10.832995 20235 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-2/data,/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-2/wal
11:41:10.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5a5e3302510a40e28f4aa8f781a651ea"
11:41:10.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 11:40:44 on dist-test-slave-h1fj"
11:41:10.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:10.836217 20235 fs_report.cc:383] FS layout report
11:41:10.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
11:41:10.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-2/wal
11:41:10.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-2/wal
11:41:10.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-2/data/data
11:41:10.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
11:41:10.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
11:41:10.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
11:41:10.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
11:41:10.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
11:41:10.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
11:41:10.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
11:41:10.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
11:41:10.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
11:41:10.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
11:41:10.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
11:41:10.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:10.898452 20235 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
11:41:10.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:10.904073 20235 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
11:41:10.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:10.904543 20235 kserver.cc:163] Server-wide thread pool size limit: 3276
11:41:10.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:10.986131 20235 rpc_server.cc:307] RPC server started. Bound to: 127.18.108.124:42121
11:41:10.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:10.986250 20328 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.18.108.124:42121 every 8 connection(s)
11:41:10.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:10.988963 20235 server_base.cc:1166] Dumped server information to /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-2/data/info.pb
11:41:10.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:10.994774 18865 external_mini_cluster.cc:1388] Started /tmp/dist-test-task4Z5jUl/build/tsan/bin/kudu as pid 20235
11:41:10.995 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
11:41:10.996 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.18.108.125:36939(127.18.108.125:36939)] connecting to peer
11:41:10.998 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.18.108.125:36939(127.18.108.125:36939)] Successfully connected to peer
11:41:11.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:11.000804 20329 sys_catalog.cc:263] Verifying existing consensus state
11:41:11.010 [DEBUG - kudu-nio-1] (SecurityContext.java:225) Refreshing Kerberos credentials...
11:41:11.010 [DEBUG - kudu-nio-1] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/krb5kdc/krb5cc
11:41:11.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:11.008671 20329 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5a5e3302510a40e28f4aa8f781a651ea: Bootstrap starting.
11:41:11.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 11:41:11 dist-test-slave-h1fj krb5kdc[18876](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790682058, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
11:41:11.015 [DEBUG - kudu-nio-1] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
11:41:11.017 [INFO - kudu-nio-1] (SecurityContext.java:259) Successfully refreshed Kerberos credentials from ticket cache
11:41:11.017 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.18.108.125
11:41:11.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 11:41:11 dist-test-slave-h1fj krb5kdc[18876](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790682058, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.18.108.125@KRBTEST.COM
11:41:11.049 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0xd93ce78b, L:/127.0.0.1:58564 - R:/127.18.108.125:36939] using SASL/GSSAPI
11:41:11.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:11.057705 20329 log.cc:824] T 00000000000000000000000000000000 P 5a5e3302510a40e28f4aa8f781a651ea: Log is configured to *not* fsync() on all Append() calls
11:41:11.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:11.080844 20329 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5a5e3302510a40e28f4aa8f781a651ea: Bootstrap replayed 1/1 log segments. Stats: ops{read=7 overwritten=0 applied=7 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
11:41:11.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 11:41:11 dist-test-slave-h1fj krb5kdc[18876](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790682068, etypes {rep=17 tkt=17 ses=17}, kudu/127.18.108.125@KRBTEST.COM for kudu/127.18.108.124@KRBTEST.COM
11:41:11.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:11.088800 20329 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5a5e3302510a40e28f4aa8f781a651ea: Bootstrap complete.
11:41:11.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:11.118963 20329 raft_consensus.cc:348] T 00000000000000000000000000000000 P 5a5e3302510a40e28f4aa8f781a651ea [term 4 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eb8f4e2d825e4e319fd704623a2750bf" member_type: VOTER last_known_addr { host: "127.18.108.126" port: 43211 } } peers { permanent_uuid: "c716745b4a1b4b39957ab73274b4df06" member_type: VOTER last_known_addr { host: "127.18.108.125" port: 36939 } } peers { permanent_uuid: "5a5e3302510a40e28f4aa8f781a651ea" member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 } }
11:41:11.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:11.122406 20329 raft_consensus.cc:729] T 00000000000000000000000000000000 P 5a5e3302510a40e28f4aa8f781a651ea [term 4 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5a5e3302510a40e28f4aa8f781a651ea, State: Initialized, Role: FOLLOWER
11:41:11.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:11.123533 20329 consensus_queue.cc:260] T 00000000000000000000000000000000 P 5a5e3302510a40e28f4aa8f781a651ea [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 7, Last appended: 4.7, Last appended by leader: 7, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eb8f4e2d825e4e319fd704623a2750bf" member_type: VOTER last_known_addr { host: "127.18.108.126" port: 43211 } } peers { permanent_uuid: "c716745b4a1b4b39957ab73274b4df06" member_type: VOTER last_known_addr { host: "127.18.108.125" port: 36939 } } peers { permanent_uuid: "5a5e3302510a40e28f4aa8f781a651ea" member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 } }
11:41:11.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:11.142853 20329 sys_catalog.cc:564] T 00000000000000000000000000000000 P 5a5e3302510a40e28f4aa8f781a651ea [sys.catalog]: configured and running, proceeding with master startup.
11:41:11.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:11.143370 20304 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 5a5e3302510a40e28f4aa8f781a651ea [term 4 FOLLOWER]: Advancing to term 5
11:41:11.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:11.141695 20336 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5a5e3302510a40e28f4aa8f781a651ea [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 4 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eb8f4e2d825e4e319fd704623a2750bf" member_type: VOTER last_known_addr { host: "127.18.108.126" port: 43211 } } peers { permanent_uuid: "c716745b4a1b4b39957ab73274b4df06" member_type: VOTER last_known_addr { host: "127.18.108.125" port: 36939 } } peers { permanent_uuid: "5a5e3302510a40e28f4aa8f781a651ea" member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 } } }
11:41:11.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:11.152585 20336 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5a5e3302510a40e28f4aa8f781a651ea [sys.catalog]: This master's current role is: FOLLOWER
11:41:11.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:11.154515 20304 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 5a5e3302510a40e28f4aa8f781a651ea [term 5 FOLLOWER]: Refusing update from remote peer c716745b4a1b4b39957ab73274b4df06: Log matching property violated. Preceding OpId in replica: term: 4 index: 7. Preceding OpId from leader: term: 5 index: 8. (index mismatch)
11:41:11.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:11.157246 20269 consensus_queue.cc:1035] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5a5e3302510a40e28f4aa8f781a651ea" member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
11:41:11.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:11.182673 20336 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5a5e3302510a40e28f4aa8f781a651ea [sys.catalog]: SysCatalogTable state changed. Reason: New leader c716745b4a1b4b39957ab73274b4df06. Latest consensus state: current_term: 5 leader_uuid: "c716745b4a1b4b39957ab73274b4df06" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eb8f4e2d825e4e319fd704623a2750bf" member_type: VOTER last_known_addr { host: "127.18.108.126" port: 43211 } } peers { permanent_uuid: "c716745b4a1b4b39957ab73274b4df06" member_type: VOTER last_known_addr { host: "127.18.108.125" port: 36939 } } peers { permanent_uuid: "5a5e3302510a40e28f4aa8f781a651ea" member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 } } }
11:41:11.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:11.183383 20336 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5a5e3302510a40e28f4aa8f781a651ea [sys.catalog]: This master's current role is: FOLLOWER
11:41:11.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:11.192961 20269 sys_catalog.cc:455] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 5 leader_uuid: "c716745b4a1b4b39957ab73274b4df06" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eb8f4e2d825e4e319fd704623a2750bf" member_type: VOTER last_known_addr { host: "127.18.108.126" port: 43211 } } peers { permanent_uuid: "c716745b4a1b4b39957ab73274b4df06" member_type: VOTER last_known_addr { host: "127.18.108.125" port: 36939 } } peers { permanent_uuid: "5a5e3302510a40e28f4aa8f781a651ea" member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 } } }
11:41:11.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:11.193634 20269 sys_catalog.cc:458] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [sys.catalog]: This master's current role is: LEADER
11:41:11.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:11.194089 20115 raft_consensus.cc:1261] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [term 5 FOLLOWER]: Refusing update from remote peer c716745b4a1b4b39957ab73274b4df06: Log matching property violated. Preceding OpId in replica: term: 4 index: 7. Preceding OpId from leader: term: 5 index: 8. (index mismatch)
11:41:11.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:11.195935 20269 consensus_queue.cc:1035] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [LEADER]: Connected to new peer: Peer: permanent_uuid: "eb8f4e2d825e4e319fd704623a2750bf" member_type: VOTER last_known_addr { host: "127.18.108.126" port: 43211 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
11:41:11.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:11.201673 20343 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5a5e3302510a40e28f4aa8f781a651ea [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 5 leader_uuid: "c716745b4a1b4b39957ab73274b4df06" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eb8f4e2d825e4e319fd704623a2750bf" member_type: VOTER last_known_addr { host: "127.18.108.126" port: 43211 } } peers { permanent_uuid: "c716745b4a1b4b39957ab73274b4df06" member_type: VOTER last_known_addr { host: "127.18.108.125" port: 36939 } } peers { permanent_uuid: "5a5e3302510a40e28f4aa8f781a651ea" member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 } } }
11:41:11.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:11.202421 20343 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5a5e3302510a40e28f4aa8f781a651ea [sys.catalog]: This master's current role is: FOLLOWER
11:41:11.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:11.228291 20346 sys_catalog.cc:455] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [sys.catalog]: SysCatalogTable state changed. Reason: New leader c716745b4a1b4b39957ab73274b4df06. Latest consensus state: current_term: 5 leader_uuid: "c716745b4a1b4b39957ab73274b4df06" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eb8f4e2d825e4e319fd704623a2750bf" member_type: VOTER last_known_addr { host: "127.18.108.126" port: 43211 } } peers { permanent_uuid: "c716745b4a1b4b39957ab73274b4df06" member_type: VOTER last_known_addr { host: "127.18.108.125" port: 36939 } } peers { permanent_uuid: "5a5e3302510a40e28f4aa8f781a651ea" member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 } } }
11:41:11.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:11.231710 20346 sys_catalog.cc:458] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [sys.catalog]: This master's current role is: FOLLOWER
11:41:11.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:11.234411 20351 sys_catalog.cc:455] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 5 leader_uuid: "c716745b4a1b4b39957ab73274b4df06" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eb8f4e2d825e4e319fd704623a2750bf" member_type: VOTER last_known_addr { host: "127.18.108.126" port: 43211 } } peers { permanent_uuid: "c716745b4a1b4b39957ab73274b4df06" member_type: VOTER last_known_addr { host: "127.18.108.125" port: 36939 } } peers { permanent_uuid: "5a5e3302510a40e28f4aa8f781a651ea" member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 } } }
11:41:11.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:11.235275 20351 sys_catalog.cc:458] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [sys.catalog]: This master's current role is: FOLLOWER
11:41:11.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:11.265133 20357 catalog_manager.cc:1259] Loaded cluster ID: 09fd2865e33b46ce9a354093f255891e
11:41:11.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:11.265379 20357 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 5a5e3302510a40e28f4aa8f781a651ea: loading cluster ID for follower catalog manager: success
11:41:11.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 11:41:11 dist-test-slave-h1fj krb5kdc[18876](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790682070, etypes {rep=17 tkt=17 ses=17}, kudu/127.18.108.124@KRBTEST.COM for kudu/127.18.108.126@KRBTEST.COM
11:41:11.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:11.280154 20357 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 5a5e3302510a40e28f4aa8f781a651ea: acquiring CA information for follower catalog manager: success
11:41:11.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:11.286453 20357 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 5a5e3302510a40e28f4aa8f781a651ea: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
11:41:11.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:11.727648 20340 sys_catalog.cc:455] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 5 leader_uuid: "c716745b4a1b4b39957ab73274b4df06" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eb8f4e2d825e4e319fd704623a2750bf" member_type: VOTER last_known_addr { host: "127.18.108.126" port: 43211 } } peers { permanent_uuid: "c716745b4a1b4b39957ab73274b4df06" member_type: VOTER last_known_addr { host: "127.18.108.125" port: 36939 } } peers { permanent_uuid: "5a5e3302510a40e28f4aa8f781a651ea" member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 } } }
11:41:11.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:11.728392 20340 sys_catalog.cc:458] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [sys.catalog]: This master's current role is: LEADER
11:41:12.052 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.18.108.126:43211
11:41:12.052 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
11:41:12.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:12.053194 18865 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task4Z5jUl/build/tsan/bin/kudu with pid 20068
11:41:12.088 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
11:41:12.088 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.18.108.125:36939
11:41:12.089 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
11:41:12.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:12.089651 18865 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task4Z5jUl/build/tsan/bin/kudu with pid 20142
11:41:12.115 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.18.108.125:36939(127.18.108.125:36939)] handling channelInactive
11:41:12.115 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.18.108.125:36939(127.18.108.125:36939)] cleaning up while in state READY due to: connection closed
11:41:12.116 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
11:41:12.117 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.18.108.124:42121
11:41:12.117 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
11:41:12.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:12.117621 18865 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task4Z5jUl/build/tsan/bin/kudu with pid 20235
11:41:12.138 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
11:41:12.138 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.18.108.126:43211
11:41:12.138 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
11:41:12.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:12.139655 18865 external_mini_cluster.cc:1326] Running /tmp/dist-test-task4Z5jUl/build/tsan/bin/kudu
11:41:12.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task4Z5jUl/build/tsan/bin/kudu
11:41:12.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-0/wal
11:41:12.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-0/data
11:41:12.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
11:41:12.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
11:41:12.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
11:41:12.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:41:12.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:41:12.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:41:12.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-0/logs
11:41:12.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-0/data/info.pb
11:41:12.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:41:12.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
11:41:12.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
11:41:12.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
11:41:12.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
11:41:12.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:41:12.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:41:12.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:41:12.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
11:41:12.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
11:41:12.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
11:41:12.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
11:41:12.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.108.126:43211
11:41:12.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.108.126
11:41:12.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36579
11:41:12.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/krb5kdc/kudu_127.18.108.126.keytab
11:41:12.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.18.108.126
11:41:12.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
11:41:12.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
11:41:12.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
11:41:12.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.108.84:34171
11:41:12.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:41:12.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:41:12.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:41:12.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
11:41:12.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.18.108.126:43211,127.18.108.125:36939,127.18.108.124:42121
11:41:12.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
11:41:12.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45823/jwks.json
11:41:12.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
11:41:12.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:41:12.425675 20366 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
11:41:12.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:41:12.426245 20366 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
11:41:12.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:41:12.426903 20366 flags.cc:425] Enabled unsafe flag: --never_fsync=true
11:41:12.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:41:12.457047 20366 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
11:41:12.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:41:12.457283 20366 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
11:41:12.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:41:12.457669 20366 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
11:41:12.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:41:12.457938 20366 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
11:41:12.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:41:12.458212 20366 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
11:41:12.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:41:12.458489 20366 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
11:41:12.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:41:12.458688 20366 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:45823/jwks.json
11:41:12.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:12.493739 20366 master_runner.cc:386] Master server non-default flags:
11:41:12.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:41:12.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.108.84:34171
11:41:12.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:41:12.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:41:12.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-0/data
11:41:12.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-0/wal
11:41:12.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
11:41:12.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
11:41:12.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.18.108.126:43211,127.18.108.125:36939,127.18.108.124:42121
11:41:12.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/krb5kdc/kudu_127.18.108.126.keytab
11:41:12.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.18.108.126
11:41:12.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:41:12.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:41:12.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
11:41:12.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.108.126:43211
11:41:12.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
11:41:12.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
11:41:12.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
11:41:12.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45823/jwks.json
11:41:12.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
11:41:12.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
11:41:12.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
11:41:12.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:41:12.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:41:12.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-0/data/info.pb
11:41:12.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.108.126
11:41:12.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36579
11:41:12.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
11:41:12.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:41:12.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
11:41:12.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
11:41:12.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:41:12.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-0/logs
11:41:12.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:41:12.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
11:41:12.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
11:41:12.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
11:41:12.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
11:41:12.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
11:41:12.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
11:41:12.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 10:52:50 UTC on ccycloud.quasar-peqjre.root.comops.site
11:41:12.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679900
11:41:12.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
11:41:12.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:12.495072 20366 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
11:41:12.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:12.496299 20366 file_cache.cc:493] Constructed file cache file cache with capacity 419430
11:41:12.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:41:12.512156 20375 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:41:13.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:41:12.513118 20373 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:41:13.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:41:13.694465 20374 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1183 milliseconds
11:41:13.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:41:12.512537 20372 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:41:13.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:13.694720 20366 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
11:41:13.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:13.699381 20366 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
11:41:13.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:13.702363 20366 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
11:41:13.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:13.703755 20366 hybrid_clock.cc:648] HybridClock initialized: now 1790682073703657 us; error 105 us; skew 500 ppm
11:41:13.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:13.704557 20366 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
11:41:13.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 11:41:13 dist-test-slave-h1fj krb5kdc[18876](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790682073, etypes {rep=17 tkt=17 ses=17}, kudu/127.18.108.126@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
11:41:13.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:13.710035 20366 init.cc:377] Logged in from keytab as kudu/127.18.108.126@KRBTEST.COM (short username kudu)
11:41:13.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:13.714632 20366 webserver.cc:460] Webserver started at http://127.18.108.126:36579/ using document root <none> and password file <none>
11:41:13.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:13.715643 20366 fs_manager.cc:359] Metadata directory not provided
11:41:13.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:13.715989 20366 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
11:41:13.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:13.723205 20366 fs_manager.cc:709] Time spent opening directory manager: real 0.004s user 0.001s sys 0.004s
11:41:13.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:13.727365 20383 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
11:41:13.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:13.728309 20366 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.004s sys 0.000s
11:41:13.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:13.728561 20366 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-0/data,/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-0/wal
11:41:13.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "eb8f4e2d825e4e319fd704623a2750bf"
11:41:13.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 11:40:40 on dist-test-slave-h1fj"
11:41:13.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:13.730764 20366 fs_report.cc:383] FS layout report
11:41:13.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
11:41:13.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-0/wal
11:41:13.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-0/wal
11:41:13.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-0/data/data
11:41:13.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
11:41:13.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
11:41:13.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
11:41:13.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
11:41:13.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
11:41:13.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
11:41:13.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
11:41:13.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
11:41:13.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
11:41:13.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
11:41:13.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
11:41:13.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:13.769136 20366 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
11:41:13.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:13.774464 20366 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
11:41:13.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:13.774843 20366 kserver.cc:163] Server-wide thread pool size limit: 3276
11:41:13.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:13.842063 20366 rpc_server.cc:307] RPC server started. Bound to: 127.18.108.126:43211
11:41:13.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:13.842193 20434 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.18.108.126:43211 every 8 connection(s)
11:41:13.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:13.844689 20366 server_base.cc:1166] Dumped server information to /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-0/data/info.pb
11:41:13.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:13.851706 18865 external_mini_cluster.cc:1388] Started /tmp/dist-test-task4Z5jUl/build/tsan/bin/kudu as pid 20366
11:41:13.852 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
11:41:13.852 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.18.108.125:36939
11:41:13.852 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
11:41:13.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:13.853663 18865 external_mini_cluster.cc:1326] Running /tmp/dist-test-task4Z5jUl/build/tsan/bin/kudu
11:41:13.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task4Z5jUl/build/tsan/bin/kudu
11:41:13.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-1/wal
11:41:13.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-1/data
11:41:13.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
11:41:13.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
11:41:13.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
11:41:13.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:41:13.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:41:13.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:41:13.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-1/logs
11:41:13.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-1/data/info.pb
11:41:13.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:41:13.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
11:41:13.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
11:41:13.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
11:41:13.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
11:41:13.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:41:13.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:41:13.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:41:13.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
11:41:13.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
11:41:13.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
11:41:13.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
11:41:13.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.108.125:36939
11:41:13.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.108.125
11:41:13.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38867
11:41:13.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/krb5kdc/kudu_127.18.108.125.keytab
11:41:13.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.18.108.125
11:41:13.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
11:41:13.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
11:41:13.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
11:41:13.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.108.84:34171
11:41:13.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:41:13.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:41:13.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:41:13.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
11:41:13.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.18.108.126:43211,127.18.108.125:36939,127.18.108.124:42121
11:41:13.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
11:41:13.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45823/jwks.json
11:41:13.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
11:41:13.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:13.855345 20435 sys_catalog.cc:263] Verifying existing consensus state
11:41:13.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:13.860733 20435 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf: Bootstrap starting.
11:41:13.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:13.896157 20435 log.cc:824] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf: Log is configured to *not* fsync() on all Append() calls
11:41:13.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:13.913975 20435 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf: Bootstrap replayed 1/1 log segments. Stats: ops{read=8 overwritten=0 applied=8 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
11:41:13.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:13.914644 20435 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf: Bootstrap complete.
11:41:13.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:13.938510 20435 raft_consensus.cc:348] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [term 5 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eb8f4e2d825e4e319fd704623a2750bf" member_type: VOTER last_known_addr { host: "127.18.108.126" port: 43211 } } peers { permanent_uuid: "c716745b4a1b4b39957ab73274b4df06" member_type: VOTER last_known_addr { host: "127.18.108.125" port: 36939 } } peers { permanent_uuid: "5a5e3302510a40e28f4aa8f781a651ea" member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 } }
11:41:13.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:13.940863 20435 raft_consensus.cc:729] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [term 5 FOLLOWER]: Becoming Follower/Learner. State: Replica: eb8f4e2d825e4e319fd704623a2750bf, State: Initialized, Role: FOLLOWER
11:41:13.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:13.941639 20435 consensus_queue.cc:260] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 8, Last appended: 5.8, Last appended by leader: 8, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eb8f4e2d825e4e319fd704623a2750bf" member_type: VOTER last_known_addr { host: "127.18.108.126" port: 43211 } } peers { permanent_uuid: "c716745b4a1b4b39957ab73274b4df06" member_type: VOTER last_known_addr { host: "127.18.108.125" port: 36939 } } peers { permanent_uuid: "5a5e3302510a40e28f4aa8f781a651ea" member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 } }
11:41:13.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:13.944295 20439 sys_catalog.cc:455] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 5 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eb8f4e2d825e4e319fd704623a2750bf" member_type: VOTER last_known_addr { host: "127.18.108.126" port: 43211 } } peers { permanent_uuid: "c716745b4a1b4b39957ab73274b4df06" member_type: VOTER last_known_addr { host: "127.18.108.125" port: 36939 } } peers { permanent_uuid: "5a5e3302510a40e28f4aa8f781a651ea" member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 } } }
11:41:13.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:13.945112 20439 sys_catalog.cc:458] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [sys.catalog]: This master's current role is: FOLLOWER
11:41:13.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:13.945870 20435 sys_catalog.cc:564] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [sys.catalog]: configured and running, proceeding with master startup.
11:41:13.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:13.982657 20450 catalog_manager.cc:1259] Loaded cluster ID: 09fd2865e33b46ce9a354093f255891e
11:41:13.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:13.982899 20450 catalog_manager.cc:1552] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf: loading cluster ID for follower catalog manager: success
11:41:13.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:13.991691 20450 catalog_manager.cc:1574] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf: acquiring CA information for follower catalog manager: success
11:41:13.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:13.998792 20450 catalog_manager.cc:1602] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
11:41:14.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 11:41:14 dist-test-slave-h1fj krb5kdc[18876](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790682073, etypes {rep=17 tkt=17 ses=17}, kudu/127.18.108.126@KRBTEST.COM for kudu/127.18.108.126@KRBTEST.COM
11:41:14.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:14.026387 20366 master_runner.cc:186] Error getting master registration for 127.18.108.125:36939: Network error: Client connection negotiation failed: client connection to 127.18.108.125:36939: connect: Connection refused (error 111),
11:41:14.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:14.045680 20366 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
11:41:14.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:41:14.245605 20437 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
11:41:14.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:41:14.246354 20437 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
11:41:14.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:41:14.247385 20437 flags.cc:425] Enabled unsafe flag: --never_fsync=true
11:41:14.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:41:14.300777 20437 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
11:41:14.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:41:14.301100 20437 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
11:41:14.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:41:14.301550 20437 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
11:41:14.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:41:14.301882 20437 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
11:41:14.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:41:14.302204 20437 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
11:41:14.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:41:14.302518 20437 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
11:41:14.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:41:14.302757 20437 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:45823/jwks.json
11:41:14.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:14.348326 20437 master_runner.cc:386] Master server non-default flags:
11:41:14.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:41:14.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.108.84:34171
11:41:14.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:41:14.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:41:14.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-1/data
11:41:14.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-1/wal
11:41:14.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
11:41:14.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
11:41:14.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.18.108.126:43211,127.18.108.125:36939,127.18.108.124:42121
11:41:14.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/krb5kdc/kudu_127.18.108.125.keytab
11:41:14.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.18.108.125
11:41:14.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:41:14.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:41:14.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
11:41:14.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.108.125:36939
11:41:14.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
11:41:14.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
11:41:14.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
11:41:14.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45823/jwks.json
11:41:14.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
11:41:14.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
11:41:14.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
11:41:14.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:41:14.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:41:14.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-1/data/info.pb
11:41:14.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.108.125
11:41:14.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38867
11:41:14.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
11:41:14.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:41:14.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
11:41:14.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
11:41:14.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:41:14.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-1/logs
11:41:14.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:41:14.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
11:41:14.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
11:41:14.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
11:41:14.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
11:41:14.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
11:41:14.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
11:41:14.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 10:52:50 UTC on ccycloud.quasar-peqjre.root.comops.site
11:41:14.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679900
11:41:14.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
11:41:14.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:14.349733 20437 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
11:41:14.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:14.350943 20437 file_cache.cc:493] Constructed file cache file cache with capacity 419430
11:41:14.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:41:14.369679 20460 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:41:15.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:15.799819 20466 raft_consensus.cc:482] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [term 5 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
11:41:15.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:41:14.373833 20462 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:41:15.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:41:15.800397 20461 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1431 milliseconds
11:41:15.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:15.800691 20437 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
11:41:15.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:15.800777 20466 raft_consensus.cc:504] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [term 5 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eb8f4e2d825e4e319fd704623a2750bf" member_type: VOTER last_known_addr { host: "127.18.108.126" port: 43211 } } peers { permanent_uuid: "c716745b4a1b4b39957ab73274b4df06" member_type: VOTER last_known_addr { host: "127.18.108.125" port: 36939 } } peers { permanent_uuid: "5a5e3302510a40e28f4aa8f781a651ea" member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 } }
11:41:15.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:41:14.370024 20459 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:41:15.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:15.808504 20437 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
11:41:15.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:15.804574 20466 leader_election.cc:288] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [CANDIDATE]: Term 6 pre-election: Requested pre-vote from peers c716745b4a1b4b39957ab73274b4df06 (127.18.108.125:36939), 5a5e3302510a40e28f4aa8f781a651ea (127.18.108.124:42121)
11:41:15.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:41:15.810961 20386 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.18.108.124:42121: connect: Connection refused (error 111)
11:41:15.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:15.813087 20437 hybrid_clock.cc:648] HybridClock initialized: now 1790682075813033 us; error 44 us; skew 500 ppm
11:41:15.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:15.814148 20437 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
11:41:15.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:41:15.815670 20384 leader_election.cc:334] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [CANDIDATE]: Term 6 pre-election: RPC error from VoteRequest() call to peer c716745b4a1b4b39957ab73274b4df06 (127.18.108.125:36939): Network error: Client connection negotiation failed: client connection to 127.18.108.125:36939: connect: Connection refused (error 111)
11:41:15.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:41:15.816241 20386 leader_election.cc:334] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [CANDIDATE]: Term 6 pre-election: RPC error from VoteRequest() call to peer 5a5e3302510a40e28f4aa8f781a651ea (127.18.108.124:42121): Network error: Client connection negotiation failed: client connection to 127.18.108.124:42121: connect: Connection refused (error 111)
11:41:15.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:15.816619 20386 leader_election.cc:302] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [CANDIDATE]: Term 6 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: eb8f4e2d825e4e319fd704623a2750bf; no voters: 5a5e3302510a40e28f4aa8f781a651ea, c716745b4a1b4b39957ab73274b4df06
11:41:15.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:15.817698 20466 raft_consensus.cc:2726] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [term 5 FOLLOWER]: Leader pre-election lost for term 6. Reason: could not achieve majority
11:41:15.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 11:41:15 dist-test-slave-h1fj krb5kdc[18876](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790682075, etypes {rep=17 tkt=17 ses=17}, kudu/127.18.108.125@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
11:41:15.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:15.820683 20437 init.cc:377] Logged in from keytab as kudu/127.18.108.125@KRBTEST.COM (short username kudu)
11:41:15.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:15.826288 20437 webserver.cc:460] Webserver started at http://127.18.108.125:38867/ using document root <none> and password file <none>
11:41:15.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:15.827093 20437 fs_manager.cc:359] Metadata directory not provided
11:41:15.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:15.827304 20437 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
11:41:15.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:15.836290 20437 fs_manager.cc:709] Time spent opening directory manager: real 0.006s user 0.007s sys 0.001s
11:41:15.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:15.841032 20476 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
11:41:15.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:15.842134 20437 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
11:41:15.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:15.842450 20437 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-1/data,/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-1/wal
11:41:15.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c716745b4a1b4b39957ab73274b4df06"
11:41:15.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 11:40:42 on dist-test-slave-h1fj"
11:41:15.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:15.844791 20437 fs_report.cc:383] FS layout report
11:41:15.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
11:41:15.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-1/wal
11:41:15.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-1/wal
11:41:15.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-1/data/data
11:41:15.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
11:41:15.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
11:41:15.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
11:41:15.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
11:41:15.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
11:41:15.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
11:41:15.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
11:41:15.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
11:41:15.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
11:41:15.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
11:41:15.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
11:41:15.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:15.887887 20437 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
11:41:15.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:15.893182 20437 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
11:41:15.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:15.893632 20437 kserver.cc:163] Server-wide thread pool size limit: 3276
11:41:15.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:15.974624 20437 rpc_server.cc:307] RPC server started. Bound to: 127.18.108.125:36939
11:41:15.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:15.974751 20527 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.18.108.125:36939 every 8 connection(s)
11:41:15.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:15.977349 20437 server_base.cc:1166] Dumped server information to /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-1/data/info.pb
11:41:15.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:15.980120 18865 external_mini_cluster.cc:1388] Started /tmp/dist-test-task4Z5jUl/build/tsan/bin/kudu as pid 20437
11:41:15.980 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
11:41:15.981 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.18.108.124:42121
11:41:15.981 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
11:41:15.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:15.982566 18865 external_mini_cluster.cc:1326] Running /tmp/dist-test-task4Z5jUl/build/tsan/bin/kudu
11:41:15.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task4Z5jUl/build/tsan/bin/kudu
11:41:15.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-2/wal
11:41:15.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-2/data
11:41:15.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
11:41:15.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
11:41:15.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
11:41:15.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:41:15.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:41:15.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:41:15.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-2/logs
11:41:15.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-2/data/info.pb
11:41:15.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:41:15.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
11:41:15.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
11:41:15.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
11:41:15.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
11:41:15.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:41:15.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:41:15.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:41:15.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
11:41:15.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
11:41:15.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
11:41:15.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
11:41:15.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.108.124:42121
11:41:15.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.108.124
11:41:15.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37519
11:41:15.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/krb5kdc/kudu_127.18.108.124.keytab
11:41:15.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.18.108.124
11:41:15.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
11:41:15.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
11:41:15.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
11:41:15.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.108.84:34171
11:41:15.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:41:15.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:41:15.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:41:15.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
11:41:15.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.18.108.126:43211,127.18.108.125:36939,127.18.108.124:42121
11:41:15.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
11:41:15.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45823/jwks.json
11:41:15.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
11:41:15.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:15.991513 20528 sys_catalog.cc:263] Verifying existing consensus state
11:41:15.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:15.997035 20528 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06: Bootstrap starting.
11:41:16.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:16.035899 20528 log.cc:824] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06: Log is configured to *not* fsync() on all Append() calls
11:41:16.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:16.054932 20528 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06: Bootstrap replayed 1/1 log segments. Stats: ops{read=8 overwritten=0 applied=8 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
11:41:16.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:16.055689 20528 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06: Bootstrap complete.
11:41:16.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:16.078612 20528 raft_consensus.cc:348] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [term 5 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eb8f4e2d825e4e319fd704623a2750bf" member_type: VOTER last_known_addr { host: "127.18.108.126" port: 43211 } } peers { permanent_uuid: "c716745b4a1b4b39957ab73274b4df06" member_type: VOTER last_known_addr { host: "127.18.108.125" port: 36939 } } peers { permanent_uuid: "5a5e3302510a40e28f4aa8f781a651ea" member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 } }
11:41:16.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:16.081048 20528 raft_consensus.cc:729] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [term 5 FOLLOWER]: Becoming Follower/Learner. State: Replica: c716745b4a1b4b39957ab73274b4df06, State: Initialized, Role: FOLLOWER
11:41:16.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:16.081835 20528 consensus_queue.cc:260] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 8, Last appended: 5.8, Last appended by leader: 8, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eb8f4e2d825e4e319fd704623a2750bf" member_type: VOTER last_known_addr { host: "127.18.108.126" port: 43211 } } peers { permanent_uuid: "c716745b4a1b4b39957ab73274b4df06" member_type: VOTER last_known_addr { host: "127.18.108.125" port: 36939 } } peers { permanent_uuid: "5a5e3302510a40e28f4aa8f781a651ea" member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 } }
11:41:16.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:16.102399 20532 sys_catalog.cc:455] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 5 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eb8f4e2d825e4e319fd704623a2750bf" member_type: VOTER last_known_addr { host: "127.18.108.126" port: 43211 } } peers { permanent_uuid: "c716745b4a1b4b39957ab73274b4df06" member_type: VOTER last_known_addr { host: "127.18.108.125" port: 36939 } } peers { permanent_uuid: "5a5e3302510a40e28f4aa8f781a651ea" member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 } } }
11:41:16.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:16.103447 20532 sys_catalog.cc:458] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [sys.catalog]: This master's current role is: FOLLOWER
11:41:16.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:16.104252 20528 sys_catalog.cc:564] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [sys.catalog]: configured and running, proceeding with master startup.
11:41:16.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:16.187086 20543 catalog_manager.cc:1259] Loaded cluster ID: 09fd2865e33b46ce9a354093f255891e
11:41:16.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:16.187362 20543 catalog_manager.cc:1552] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06: loading cluster ID for follower catalog manager: success
11:41:16.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 11:41:16 dist-test-slave-h1fj krb5kdc[18876](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790682075, etypes {rep=17 tkt=17 ses=17}, kudu/127.18.108.125@KRBTEST.COM for kudu/127.18.108.126@KRBTEST.COM
11:41:16.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:16.210027 20543 catalog_manager.cc:1574] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06: acquiring CA information for follower catalog manager: success
11:41:16.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:16.215315 20543 catalog_manager.cc:1602] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
11:41:16.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:16.285292 20437 master_runner.cc:186] Error getting master registration for 127.18.108.124:42121: Network error: Client connection negotiation failed: client connection to 127.18.108.124:42121: connect: Connection refused (error 111),
11:41:16.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:16.316475 20437 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
11:41:16.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:41:16.379057 20530 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
11:41:16.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:41:16.379599 20530 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
11:41:16.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:41:16.380152 20530 flags.cc:425] Enabled unsafe flag: --never_fsync=true
11:41:16.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:41:16.410784 20530 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
11:41:16.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:41:16.410984 20530 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
11:41:16.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:41:16.411300 20530 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
11:41:16.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:41:16.411516 20530 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
11:41:16.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:41:16.411700 20530 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
11:41:16.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:41:16.411880 20530 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
11:41:16.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:41:16.412019 20530 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:45823/jwks.json
11:41:16.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:16.447220 20530 master_runner.cc:386] Master server non-default flags:
11:41:16.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:41:16.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.108.84:34171
11:41:16.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:41:16.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:41:16.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-2/data
11:41:16.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-2/wal
11:41:16.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
11:41:16.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
11:41:16.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.18.108.126:43211,127.18.108.125:36939,127.18.108.124:42121
11:41:16.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/krb5kdc/kudu_127.18.108.124.keytab
11:41:16.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.18.108.124
11:41:16.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:41:16.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:41:16.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
11:41:16.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.108.124:42121
11:41:16.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
11:41:16.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
11:41:16.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
11:41:16.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45823/jwks.json
11:41:16.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
11:41:16.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
11:41:16.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
11:41:16.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:41:16.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:41:16.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-2/data/info.pb
11:41:16.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.108.124
11:41:16.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37519
11:41:16.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
11:41:16.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:41:16.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
11:41:16.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
11:41:16.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:41:16.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-2/logs
11:41:16.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:41:16.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
11:41:16.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
11:41:16.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
11:41:16.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
11:41:16.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
11:41:16.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
11:41:16.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 10:52:50 UTC on ccycloud.quasar-peqjre.root.comops.site
11:41:16.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679900
11:41:16.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
11:41:16.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:16.448690 20530 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
11:41:16.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:16.449828 20530 file_cache.cc:493] Constructed file cache file cache with capacity 419430
11:41:16.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:41:16.467974 20554 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:41:17.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:17.536182 20559 raft_consensus.cc:482] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [term 5 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
11:41:17.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:17.537091 20559 raft_consensus.cc:504] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [term 5 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eb8f4e2d825e4e319fd704623a2750bf" member_type: VOTER last_known_addr { host: "127.18.108.126" port: 43211 } } peers { permanent_uuid: "c716745b4a1b4b39957ab73274b4df06" member_type: VOTER last_known_addr { host: "127.18.108.125" port: 36939 } } peers { permanent_uuid: "5a5e3302510a40e28f4aa8f781a651ea" member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 } }
11:41:17.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:17.554651 20559 leader_election.cc:288] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [CANDIDATE]: Term 6 pre-election: Requested pre-vote from peers c716745b4a1b4b39957ab73274b4df06 (127.18.108.125:36939), 5a5e3302510a40e28f4aa8f781a651ea (127.18.108.124:42121)
11:41:17.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:41:17.570842 20386 leader_election.cc:334] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [CANDIDATE]: Term 6 pre-election: RPC error from VoteRequest() call to peer 5a5e3302510a40e28f4aa8f781a651ea (127.18.108.124:42121): Network error: Client connection negotiation failed: client connection to 127.18.108.124:42121: connect: Connection refused (error 111)
11:41:17.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:17.584524 20503 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "eb8f4e2d825e4e319fd704623a2750bf" candidate_term: 6 candidate_status { last_received { term: 5 index: 8 } } ignore_live_leader: false dest_uuid: "c716745b4a1b4b39957ab73274b4df06" is_pre_election: true
11:41:17.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:17.585472 20503 raft_consensus.cc:2454] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [term 5 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate eb8f4e2d825e4e319fd704623a2750bf in term 5.
11:41:17.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:17.586735 20384 leader_election.cc:302] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [CANDIDATE]: Term 6 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: c716745b4a1b4b39957ab73274b4df06, eb8f4e2d825e4e319fd704623a2750bf; no voters: 5a5e3302510a40e28f4aa8f781a651ea
11:41:17.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:17.587888 20559 raft_consensus.cc:2781] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [term 5 FOLLOWER]: Leader pre-election won for term 6
11:41:17.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:17.588220 20559 raft_consensus.cc:482] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [term 5 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
11:41:17.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:17.588519 20559 raft_consensus.cc:3037] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [term 5 FOLLOWER]: Advancing to term 6
11:41:17.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:17.597707 20559 raft_consensus.cc:504] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [term 6 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eb8f4e2d825e4e319fd704623a2750bf" member_type: VOTER last_known_addr { host: "127.18.108.126" port: 43211 } } peers { permanent_uuid: "c716745b4a1b4b39957ab73274b4df06" member_type: VOTER last_known_addr { host: "127.18.108.125" port: 36939 } } peers { permanent_uuid: "5a5e3302510a40e28f4aa8f781a651ea" member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 } }
11:41:17.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:17.600553 20503 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "eb8f4e2d825e4e319fd704623a2750bf" candidate_term: 6 candidate_status { last_received { term: 5 index: 8 } } ignore_live_leader: false dest_uuid: "c716745b4a1b4b39957ab73274b4df06"
11:41:17.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:17.601111 20503 raft_consensus.cc:3037] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [term 5 FOLLOWER]: Advancing to term 6
11:41:17.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:17.610522 20503 raft_consensus.cc:2454] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [term 6 FOLLOWER]: Leader election vote request: Granting yes vote for candidate eb8f4e2d825e4e319fd704623a2750bf in term 6.
11:41:17.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:41:17.615062 20386 leader_election.cc:334] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [CANDIDATE]: Term 6 election: RPC error from VoteRequest() call to peer 5a5e3302510a40e28f4aa8f781a651ea (127.18.108.124:42121): Network error: Client connection negotiation failed: client connection to 127.18.108.124:42121: connect: Connection refused (error 111)
11:41:17.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:17.613368 20559 leader_election.cc:288] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [CANDIDATE]: Term 6 election: Requested vote from peers c716745b4a1b4b39957ab73274b4df06 (127.18.108.125:36939), 5a5e3302510a40e28f4aa8f781a651ea (127.18.108.124:42121)
11:41:17.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:17.616778 20384 leader_election.cc:302] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [CANDIDATE]: Term 6 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: c716745b4a1b4b39957ab73274b4df06, eb8f4e2d825e4e319fd704623a2750bf; no voters: 5a5e3302510a40e28f4aa8f781a651ea
11:41:17.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:17.617873 20559 raft_consensus.cc:2781] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [term 6 FOLLOWER]: Leader election won for term 6
11:41:17.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:17.627674 20559 raft_consensus.cc:686] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [term 6 LEADER]: Becoming Leader. State: Replica: eb8f4e2d825e4e319fd704623a2750bf, State: Running, Role: LEADER
11:41:17.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:17.628778 20559 consensus_queue.cc:237] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 5.8, Last appended by leader: 8, Current term: 6, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eb8f4e2d825e4e319fd704623a2750bf" member_type: VOTER last_known_addr { host: "127.18.108.126" port: 43211 } } peers { permanent_uuid: "c716745b4a1b4b39957ab73274b4df06" member_type: VOTER last_known_addr { host: "127.18.108.125" port: 36939 } } peers { permanent_uuid: "5a5e3302510a40e28f4aa8f781a651ea" member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 } }
11:41:17.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:17.645529 20564 sys_catalog.cc:455] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [sys.catalog]: SysCatalogTable state changed. Reason: New leader eb8f4e2d825e4e319fd704623a2750bf. Latest consensus state: current_term: 6 leader_uuid: "eb8f4e2d825e4e319fd704623a2750bf" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eb8f4e2d825e4e319fd704623a2750bf" member_type: VOTER last_known_addr { host: "127.18.108.126" port: 43211 } } peers { permanent_uuid: "c716745b4a1b4b39957ab73274b4df06" member_type: VOTER last_known_addr { host: "127.18.108.125" port: 36939 } } peers { permanent_uuid: "5a5e3302510a40e28f4aa8f781a651ea" member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 } } }
11:41:17.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:17.646452 20564 sys_catalog.cc:458] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [sys.catalog]: This master's current role is: LEADER
11:41:17.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:17.657524 20566 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
11:41:17.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:17.665242 20566 catalog_manager.cc:1484] Initializing Kudu cluster ID...
11:41:17.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:17.669071 20566 catalog_manager.cc:1259] Loaded cluster ID: 09fd2865e33b46ce9a354093f255891e
11:41:17.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:17.669402 20566 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
11:41:17.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:17.674088 20566 catalog_manager.cc:1504] Loading token signing keys...
11:41:17.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:17.677902 20566 catalog_manager.cc:5879] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf: Loaded TSK: 0
11:41:17.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:17.679593 20566 catalog_manager.cc:1514] Initializing in-progress tserver states...
11:41:17.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:41:16.471223 20555 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:41:17.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:41:17.862983 20553 debug-util.cc:398] Leaking SignalData structure 0x7b0800014920 after lost signal to thread 20530
11:41:17.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:41:17.982527 20530 thread.cc:636] GCE (cloud detector) Time spent creating pthread: real 1.515s user 0.585s sys 0.900s
11:41:17.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:41:17.984644 20530 thread.cc:602] GCE (cloud detector) Time spent starting thread: real 1.517s user 0.585s sys 0.900s
11:41:17.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:17.995694 20530 server_base.cc:1034] running on GCE node
11:41:17.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:41:17.997370 20569 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:41:17.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:17.998734 20530 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
11:41:18.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:18.001641 20530 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
11:41:18.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:18.003055 20530 hybrid_clock.cc:648] HybridClock initialized: now 1790682078002974 us; error 73 us; skew 500 ppm
11:41:18.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:18.004127 20530 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
11:41:18.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 11:41:18 dist-test-slave-h1fj krb5kdc[18876](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790682078, etypes {rep=17 tkt=17 ses=17}, kudu/127.18.108.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
11:41:18.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:18.010613 20530 init.cc:377] Logged in from keytab as kudu/127.18.108.124@KRBTEST.COM (short username kudu)
11:41:18.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:18.015892 20530 webserver.cc:460] Webserver started at http://127.18.108.124:37519/ using document root <none> and password file <none>
11:41:18.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:18.016829 20530 fs_manager.cc:359] Metadata directory not provided
11:41:18.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:18.017033 20530 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
11:41:18.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:18.024704 20530 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.004s sys 0.000s
11:41:18.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:18.029059 20575 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
11:41:18.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:18.030714 20530 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.004s sys 0.000s
11:41:18.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:18.032019 20503 raft_consensus.cc:1261] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [term 6 FOLLOWER]: Refusing update from remote peer eb8f4e2d825e4e319fd704623a2750bf: Log matching property violated. Preceding OpId in replica: term: 5 index: 8. Preceding OpId from leader: term: 6 index: 9. (index mismatch)
11:41:18.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:18.031030 20530 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-2/data,/tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-2/wal
11:41:18.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5a5e3302510a40e28f4aa8f781a651ea"
11:41:18.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 11:40:44 on dist-test-slave-h1fj"
11:41:18.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:18.033597 20564 consensus_queue.cc:1035] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [LEADER]: Connected to new peer: Peer: permanent_uuid: "c716745b4a1b4b39957ab73274b4df06" member_type: VOTER last_known_addr { host: "127.18.108.125" port: 36939 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
11:41:18.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:18.034047 20530 fs_report.cc:383] FS layout report
11:41:18.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
11:41:18.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-2/wal
11:41:18.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-2/wal
11:41:18.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-2/data/data
11:41:18.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
11:41:18.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
11:41:18.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
11:41:18.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
11:41:18.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
11:41:18.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
11:41:18.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
11:41:18.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
11:41:18.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
11:41:18.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
11:41:18.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
11:41:18.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:41:18.048007 20386 consensus_peers.cc:487] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf -> Peer 5a5e3302510a40e28f4aa8f781a651ea (127.18.108.124:42121): Couldn't send request to peer 5a5e3302510a40e28f4aa8f781a651ea. Status: Network error: Client connection negotiation failed: client connection to 127.18.108.124:42121: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
11:41:18.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:18.051391 20576 sys_catalog.cc:455] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [sys.catalog]: SysCatalogTable state changed. Reason: New leader eb8f4e2d825e4e319fd704623a2750bf. Latest consensus state: current_term: 6 leader_uuid: "eb8f4e2d825e4e319fd704623a2750bf" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eb8f4e2d825e4e319fd704623a2750bf" member_type: VOTER last_known_addr { host: "127.18.108.126" port: 43211 } } peers { permanent_uuid: "c716745b4a1b4b39957ab73274b4df06" member_type: VOTER last_known_addr { host: "127.18.108.125" port: 36939 } } peers { permanent_uuid: "5a5e3302510a40e28f4aa8f781a651ea" member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 } } }
11:41:18.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:18.052186 20576 sys_catalog.cc:458] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [sys.catalog]: This master's current role is: FOLLOWER
11:41:18.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:18.054227 20559 sys_catalog.cc:455] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 6 leader_uuid: "eb8f4e2d825e4e319fd704623a2750bf" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eb8f4e2d825e4e319fd704623a2750bf" member_type: VOTER last_known_addr { host: "127.18.108.126" port: 43211 } } peers { permanent_uuid: "c716745b4a1b4b39957ab73274b4df06" member_type: VOTER last_known_addr { host: "127.18.108.125" port: 36939 } } peers { permanent_uuid: "5a5e3302510a40e28f4aa8f781a651ea" member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 } } }
11:41:18.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:18.054864 20559 sys_catalog.cc:458] T 00000000000000000000000000000000 P eb8f4e2d825e4e319fd704623a2750bf [sys.catalog]: This master's current role is: LEADER
11:41:18.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:18.061345 20576 sys_catalog.cc:455] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 6 leader_uuid: "eb8f4e2d825e4e319fd704623a2750bf" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eb8f4e2d825e4e319fd704623a2750bf" member_type: VOTER last_known_addr { host: "127.18.108.126" port: 43211 } } peers { permanent_uuid: "c716745b4a1b4b39957ab73274b4df06" member_type: VOTER last_known_addr { host: "127.18.108.125" port: 36939 } } peers { permanent_uuid: "5a5e3302510a40e28f4aa8f781a651ea" member_type: VOTER last_known_addr { host: "127.18.108.124" port: 42121 } } }
11:41:18.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:18.061995 20576 sys_catalog.cc:458] T 00000000000000000000000000000000 P c716745b4a1b4b39957ab73274b4df06 [sys.catalog]: This master's current role is: FOLLOWER
11:41:18.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:18.081039 20530 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
11:41:18.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:18.087992 20530 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
11:41:18.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:18.088519 20530 kserver.cc:163] Server-wide thread pool size limit: 3276
11:41:18.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:18.170639 20530 rpc_server.cc:307] RPC server started. Bound to: 127.18.108.124:42121
11:41:18.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:18.170781 20630 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.18.108.124:42121 every 8 connection(s)
11:41:18.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:18.174232 20530 server_base.cc:1166] Dumped server information to /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/master-2/data/info.pb
11:41:18.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:18.180712 18865 external_mini_cluster.cc:1388] Started /tmp/dist-test-task4Z5jUl/build/tsan/bin/kudu as pid 20530
11:41:18.181 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
11:41:18.181 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.18.108.125:36939(127.18.108.125:36939)] connecting to peer
11:41:18.184 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.18.108.125:36939(127.18.108.125:36939)] Successfully connected to peer
11:41:18.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:18.186823 20631 sys_catalog.cc:263] Verifying existing consensus state
11:41:18.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:18.194754 20631 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5a5e3302510a40e28f4aa8f781a651ea: Bootstrap starting.
11:41:18.199 [DEBUG - kudu-nio-0] (SecurityContext.java:225) Refreshing Kerberos credentials...
11:41:18.200 [DEBUG - kudu-nio-0] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-task4Z5jUl/test-tmp/mini-kudu-cluster9166182393060915213/krb5kdc/krb5cc
11:41:18.202 [DEBUG - kudu-nio-0] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
11:41:18.202 [WARN - kudu-nio-0] (SecurityContext.java:234) Tried to refresh Kerberos credentials but was unable to re-login from ticket cache
11:41:18.202 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.18.108.125:36939(127.18.108.125:36939)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
11:41:18.204 [DEBUG - main] (Connection.java:575) [peer master-127.18.108.124:42121(127.18.108.124:42121)] disconnecting while in state TERMINATED
11:41:18.204 [DEBUG - main] (Connection.java:575) [peer master-127.18.108.126:43211(127.18.108.126:43211)] disconnecting while in state TERMINATED
11:41:18.204 [DEBUG - main] (Connection.java:575) [peer master-127.18.108.125:36939(127.18.108.125:36939)] disconnecting while in state TERMINATED
11:41:18.205 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2704) Releasing all remaining resources
11:41:18.207 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.18.108.125:36939(127.18.108.125:36939)] handling channelInactive
11:41:18.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 11:41:18.211973 20633 negotiation.cc:337] Failed RPC negotiation. Trace:
11:41:18.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0929 11:41:18.184310 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:58608 (local address 127.18.108.125:36939)
11:41:18.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0929 11:41:18.197913 (+ 13603us) server_negotiation.cc:206] Beginning negotiation
11:41:18.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0929 11:41:18.197943 (+ 30us) server_negotiation.cc:399] Waiting for connection header
11:41:18.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0929 11:41:18.197993 (+ 50us) server_negotiation.cc:407] Connection header received
11:41:18.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0929 11:41:18.198141 (+ 148us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
11:41:18.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0929 11:41:18.198165 (+ 24us) server_negotiation.cc:461] Received NEGOTIATE request from client
11:41:18.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0929 11:41:18.198388 (+ 223us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
11:41:18.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0929 11:41:18.198797 (+ 409us) server_negotiation.cc:226] Negotiated authn=SASL
11:41:18.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0929 11:41:18.205861 (+ 7064us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:58608: BlockingRecv error: recv got EOF from 127.0.0.1:58608 (error 108)
11:41:18.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":13186,"thread_start_us":14233,"threads_started":1}
11:41:18.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:18.221979 18865 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task4Z5jUl/build/tsan/bin/kudu with pid 20366
11:41:18.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:18.244138 20631 log.cc:824] T 00000000000000000000000000000000 P 5a5e3302510a40e28f4aa8f781a651ea: Log is configured to *not* fsync() on all Append() calls
11:41:18.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:18.259459 18865 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task4Z5jUl/build/tsan/bin/kudu with pid 20437
11:41:18.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:18.269771 20631 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5a5e3302510a40e28f4aa8f781a651ea: Bootstrap replayed 1/1 log segments. Stats: ops{read=8 overwritten=0 applied=8 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
11:41:18.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:18.270642 20631 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5a5e3302510a40e28f4aa8f781a651ea: Bootstrap complete.
11:41:18.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:18.285099 18865 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task4Z5jUl/build/tsan/bin/kudu with pid 20530
11:41:19.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-29T11:41:19Z chronyd exiting
11:41:20.377 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
11:41:20.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:20.378973 18141 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task4Z5jUl/build/tsan/bin/kudu with pid 18421
11:41:20.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:20.399979 18141 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task4Z5jUl/build/tsan/bin/kudu with pid 18580
11:41:20.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:20.419476 18141 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task4Z5jUl/build/tsan/bin/kudu with pid 18723
11:41:20.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:20.442274 18141 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task4Z5jUl/build/tsan/bin/kudu with pid 18153
11:41:20.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:20.465643 18141 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task4Z5jUl/build/tsan/bin/kudu with pid 18223
11:41:20.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 11:41:20.489996 18141 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task4Z5jUl/build/tsan/bin/kudu with pid 18298
11:41:21.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-29T11:41:21Z chronyd exiting
11:41:22.553 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testRenewAndReacquireKeberosCredentials: failed attempt 1
org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testRenewAndReacquireKeberosCredentials(TestSecurity.java:346) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.Negotiator.chooseAndInitializeSaslMech(Negotiator.java:565) ~[main/:?]
at org.apache.kudu.client.Negotiator.handleNegotiateResponse(Negotiator.java:433) ~[main/:?]
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:367) ~[main/:?]
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:94) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testRenewAndReacquireKeberosCredentials(TestSecurity.java:346)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.Negotiator.chooseAndInitializeSaslMech(Negotiator.java:565)
at org.apache.kudu.client.Negotiator.handleNegotiateResponse(Negotiator.java:433)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:367)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:94)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
11:41:23.605 [ERROR - main] (RetryRule.java:253) org.apache.kudu.client.TestSecurity.testRenewAndReacquireKeberosCredentials: giving up after 1 attempts
11:41:23.605 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testRenewAndReacquireKeberosCredentials: failed attempt 1
org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testRenewAndReacquireKeberosCredentials(TestSecurity.java:346) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.Negotiator.chooseAndInitializeSaslMech(Negotiator.java:565) ~[main/:?]
at org.apache.kudu.client.Negotiator.handleNegotiateResponse(Negotiator.java:433) ~[main/:?]
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:367) ~[main/:?]
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:94) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testRenewAndReacquireKeberosCredentials(TestSecurity.java:346)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.Negotiator.chooseAndInitializeSaslMech(Negotiator.java:565)
at org.apache.kudu.client.Negotiator.handleNegotiateResponse(Negotiator.java:433)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:367)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:94)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more