20:08:57.859 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
20:08:57.870 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskQoB_wm/test-tmp
20:08:57.873 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
20:08:57.876 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskQoB_wm/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
20:08:58.084 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster934885147846391950"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
20:08:58.259 [DEBUG - main] (MiniKuduCluster.java:195) Response:
20:08:58.260 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
20:08:58.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:08:58.264288 19134 mini_oidc.cc:152] Starting JWKS server
20:08:58.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:08:58.268006 19134 webserver.cc:460] Webserver started at http://127.0.0.1:35579/ using document root <none> and password file <none>
20:08:58.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:08:58.268602 19134 mini_oidc.cc:189] Starting OIDC Discovery server
20:08:58.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:08:58.270440 19134 webserver.cc:460] Webserver started at http://127.0.0.1:42107/ using document root <none> and password file <none>
20:08:58.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-27T20:08:58Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
20:08:58.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-27T20:08:58Z Disabled control of system clock
20:08:58.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:08:58.318465 19134 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskQoB_wm/build/tsan/bin/kudu
20:08:58.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskQoB_wm/build/tsan/bin/kudu
20:08:58.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster934885147846391950/master-0/wal
20:08:58.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster934885147846391950/master-0/data
20:08:58.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:08:58.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:08:58.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:08:58.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:08:58.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:08:58.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:08:58.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster934885147846391950/master-0/logs
20:08:58.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster934885147846391950/master-0/data/info.pb
20:08:58.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:08:58.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:08:58.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:08:58.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:08:58.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:08:58.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:08:58.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:08:58.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:08:58.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:08:58.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:08:58.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:08:58.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:08:58.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.175.190:35215
20:08:58.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.175.190
20:08:58.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:08:58.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.175.148:37841
20:08:58.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:08:58.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:08:58.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:08:58.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:08:58.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.18.175.190:35215,127.18.175.189:45157,127.18.175.188:46095
20:08:58.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:08:58.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35579/jwks.json with env {}
20:08:58.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:08:58.610639 19146 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:08:58.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:08:58.611212 19146 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:08:58.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:08:58.611865 19146 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:08:58.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:08:58.642313 19146 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
20:08:58.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:08:58.642599 19146 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:08:58.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:08:58.642788 19146 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
20:08:58.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:08:58.643056 19146 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
20:08:58.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:08:58.643347 19146 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
20:08:58.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:08:58.643579 19146 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:35579/jwks.json
20:08:58.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:08:58.677577 19146 master_runner.cc:386] Master server non-default flags:
20:08:58.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:08:58.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.175.148:37841
20:08:58.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:08:58.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:08:58.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster934885147846391950/master-0/data
20:08:58.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster934885147846391950/master-0/wal
20:08:58.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:08:58.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.18.175.190:35215,127.18.175.189:45157,127.18.175.188:46095
20:08:58.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:08:58.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:08:58.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:08:58.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.175.190:35215
20:08:58.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:08:58.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:08:58.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:08:58.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35579/jwks.json
20:08:58.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:08:58.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:08:58.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster934885147846391950/master-0/data/info.pb
20:08:58.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.175.190
20:08:58.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:08:58.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:08:58.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:08:58.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:08:58.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:08:58.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:08:58.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster934885147846391950/master-0/logs
20:08:58.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:08:58.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:08:58.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:08:58.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:08:58.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
20:08:58.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
20:08:58.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:08:58.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 19:19:31 UTC on ccycloud.quasar-tlrbor.root.comops.site
20:08:58.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676273
20:08:58.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:08:58.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:08:58.679033 19146 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:08:58.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:08:58.681170 19146 file_cache.cc:493] Constructed file cache file cache with capacity 419430
20:08:58.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:08:58.699342 19155 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:08:59.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:08:58.699445 19153 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:08:59.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:08:58.700467 19146 server_base.cc:1034] running on GCE node
20:08:59.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:08:58.699396 19152 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:08:59.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:08:59.884853 19146 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:08:59.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:08:59.888085 19146 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:08:59.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:08:59.889539 19146 hybrid_clock.cc:648] HybridClock initialized: now 1790539739889495 us; error 61 us; skew 500 ppm
20:08:59.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:08:59.890328 19146 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:08:59.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:08:59.896678 19146 webserver.cc:460] Webserver started at http://127.18.175.190:46121/ using document root <none> and password file <none>
20:08:59.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:08:59.897544 19146 fs_manager.cc:359] Metadata directory not provided
20:08:59.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:08:59.897719 19146 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:08:59.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:08:59.898084 19146 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
20:08:59.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:08:59.902972 19146 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster934885147846391950/master-0/data/instance:
20:08:59.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fd68912fb21c4af8803a4eea5cb9b2de"
20:08:59.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 20:08:59 on dist-test-slave-6w59"
20:08:59.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:08:59.904140 19146 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster934885147846391950/master-0/wal/instance:
20:08:59.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fd68912fb21c4af8803a4eea5cb9b2de"
20:08:59.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 20:08:59 on dist-test-slave-6w59"
20:08:59.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:08:59.911458 19146 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.007s sys 0.000s
20:08:59.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:08:59.916790 19162 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:08:59.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:08:59.917907 19146 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.001s sys 0.004s
20:08:59.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:08:59.918196 19146 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster934885147846391950/master-0/data,/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster934885147846391950/master-0/wal
20:08:59.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fd68912fb21c4af8803a4eea5cb9b2de"
20:08:59.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 20:08:59 on dist-test-slave-6w59"
20:08:59.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:08:59.918572 19146 fs_report.cc:383] FS layout report
20:08:59.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:08:59.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster934885147846391950/master-0/wal
20:08:59.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster934885147846391950/master-0/wal
20:08:59.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster934885147846391950/master-0/data/data
20:08:59.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:08:59.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:08:59.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:08:59.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:08:59.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:08:59.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:08:59.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:08:59.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:08:59.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:08:59.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:08:59.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:08:59.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:08:59.960661 19146 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
20:08:59.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:08:59.962073 19146 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:08:59.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:08:59.962462 19146 kserver.cc:163] Server-wide thread pool size limit: 3276
20:09:00.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:00.031136 19146 rpc_server.cc:307] RPC server started. Bound to: 127.18.175.190:35215
20:09:00.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:00.031227 19214 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.18.175.190:35215 every 8 connection(s)
20:09:00.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:00.033674 19146 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster934885147846391950/master-0/data/info.pb
20:09:00.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:00.037839 19134 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskQoB_wm/build/tsan/bin/kudu as pid 19146
20:09:00.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:00.038432 19134 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster934885147846391950/master-0/wal/instance
20:09:00.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:00.040086 19215 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
20:09:00.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:00.044109 19134 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskQoB_wm/build/tsan/bin/kudu
20:09:00.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskQoB_wm/build/tsan/bin/kudu
20:09:00.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster934885147846391950/master-1/wal
20:09:00.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster934885147846391950/master-1/data
20:09:00.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:09:00.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:09:00.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:09:00.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:09:00.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:09:00.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:09:00.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster934885147846391950/master-1/logs
20:09:00.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster934885147846391950/master-1/data/info.pb
20:09:00.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:09:00.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:09:00.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:09:00.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:09:00.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:09:00.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:09:00.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:09:00.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:09:00.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:09:00.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:09:00.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:09:00.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:09:00.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.175.189:45157
20:09:00.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.175.189
20:09:00.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:09:00.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.175.148:37841
20:09:00.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:09:00.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:09:00.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:09:00.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:09:00.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.18.175.190:35215,127.18.175.189:45157,127.18.175.188:46095
20:09:00.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:09:00.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35579/jwks.json with env {}
20:09:00.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:00.060811 19215 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.18.175.190" port: 35215 } has no permanent_uuid. Determining permanent_uuid...
20:09:00.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:00.092799 19215 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.18.175.189" port: 45157 } has no permanent_uuid. Determining permanent_uuid...
20:09:00.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:00.095228 19164 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.18.175.189:45157: connect: Connection refused (error 111)
20:09:00.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:00.099380 19215 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.18.175.189:45157: Network error: Client connection negotiation failed: client connection to 127.18.175.189:45157: connect: Connection refused (error 111)
20:09:00.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:00.148974 19215 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.175.189" port: 45157 } attempt: 1
20:09:00.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:00.153235 19215 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.18.175.189:45157: Network error: Client connection negotiation failed: client connection to 127.18.175.189:45157: connect: Connection refused (error 111)
20:09:00.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:00.221622 19215 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.175.189" port: 45157 } attempt: 2
20:09:00.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:00.225307 19215 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.18.175.189:45157: Network error: Client connection negotiation failed: client connection to 127.18.175.189:45157: connect: Connection refused (error 111)
20:09:00.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:00.316725 19215 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.175.189" port: 45157 } attempt: 3
20:09:00.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:00.320492 19215 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.18.175.189:45157: Network error: Client connection negotiation failed: client connection to 127.18.175.189:45157: connect: Connection refused (error 111)
20:09:00.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:00.362504 19217 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:09:00.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:00.363125 19217 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:09:00.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:00.363749 19217 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:09:00.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:00.393953 19217 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
20:09:00.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:00.394227 19217 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:09:00.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:00.394410 19217 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
20:09:00.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:00.394598 19217 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
20:09:00.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:00.394769 19217 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
20:09:00.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:00.394960 19217 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:35579/jwks.json
20:09:00.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:00.429126 19217 master_runner.cc:386] Master server non-default flags:
20:09:00.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:09:00.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.175.148:37841
20:09:00.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:09:00.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:09:00.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster934885147846391950/master-1/data
20:09:00.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster934885147846391950/master-1/wal
20:09:00.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:09:00.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.18.175.190:35215,127.18.175.189:45157,127.18.175.188:46095
20:09:00.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:09:00.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:09:00.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:09:00.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.175.189:45157
20:09:00.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:09:00.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:09:00.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:09:00.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35579/jwks.json
20:09:00.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:09:00.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:09:00.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster934885147846391950/master-1/data/info.pb
20:09:00.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.175.189
20:09:00.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:09:00.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:09:00.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:09:00.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:09:00.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:09:00.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:09:00.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster934885147846391950/master-1/logs
20:09:00.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:09:00.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:09:00.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:09:00.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:09:00.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
20:09:00.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
20:09:00.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:09:00.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 19:19:31 UTC on ccycloud.quasar-tlrbor.root.comops.site
20:09:00.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676273
20:09:00.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:09:00.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:00.430437 19217 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:09:00.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:00.432613 19217 file_cache.cc:493] Constructed file cache file cache with capacity 419430
20:09:00.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:00.450492 19229 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:09:00.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:00.463871 19215 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.175.189" port: 45157 } attempt: 4
20:09:00.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:00.467546 19215 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.18.175.189:45157: Network error: Client connection negotiation failed: client connection to 127.18.175.189:45157: connect: Connection refused (error 111)
20:09:00.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:00.766943 19215 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.175.189" port: 45157 } attempt: 5
20:09:00.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:00.771464 19215 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.18.175.189:45157: Network error: Client connection negotiation failed: client connection to 127.18.175.189:45157: connect: Connection refused (error 111)
20:09:01.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:01.318959 19215 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.175.189" port: 45157 } attempt: 6
20:09:01.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:01.326524 19215 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.18.175.189:45157: Network error: Client connection negotiation failed: client connection to 127.18.175.189:45157: connect: Connection refused (error 111)
20:09:01.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:00.450512 19227 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:09:01.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:00.450904 19226 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:09:01.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:00.452188 19217 server_base.cc:1034] running on GCE node
20:09:01.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:01.641336 19217 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:09:01.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:01.644507 19217 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:09:01.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:01.645923 19217 hybrid_clock.cc:648] HybridClock initialized: now 1790539741645879 us; error 65 us; skew 500 ppm
20:09:01.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:01.646951 19217 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:09:01.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:01.653558 19217 webserver.cc:460] Webserver started at http://127.18.175.189:33577/ using document root <none> and password file <none>
20:09:01.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:01.654469 19217 fs_manager.cc:359] Metadata directory not provided
20:09:01.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:01.654659 19217 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:09:01.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:01.655102 19217 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
20:09:01.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:01.659929 19217 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster934885147846391950/master-1/data/instance:
20:09:01.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ffc88e8c26024be18ba7bfb1d890fe67"
20:09:01.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 20:09:01 on dist-test-slave-6w59"
20:09:01.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:01.660998 19217 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster934885147846391950/master-1/wal/instance:
20:09:01.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ffc88e8c26024be18ba7bfb1d890fe67"
20:09:01.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 20:09:01 on dist-test-slave-6w59"
20:09:01.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:01.671814 19217 fs_manager.cc:691] Time spent creating directory manager: real 0.010s user 0.006s sys 0.004s
20:09:01.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:01.677073 19238 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:09:01.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:01.678180 19217 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.004s sys 0.001s
20:09:01.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:01.678450 19217 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster934885147846391950/master-1/data,/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster934885147846391950/master-1/wal
20:09:01.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ffc88e8c26024be18ba7bfb1d890fe67"
20:09:01.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 20:09:01 on dist-test-slave-6w59"
20:09:01.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:01.678754 19217 fs_report.cc:383] FS layout report
20:09:01.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:09:01.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster934885147846391950/master-1/wal
20:09:01.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster934885147846391950/master-1/wal
20:09:01.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster934885147846391950/master-1/data/data
20:09:01.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:09:01.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:09:01.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:09:01.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:09:01.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:09:01.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:09:01.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:09:01.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:09:01.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:09:01.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:09:01.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:09:01.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:01.729489 19217 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
20:09:01.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:01.731755 19217 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:09:01.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:01.732393 19217 kserver.cc:163] Server-wide thread pool size limit: 3276
20:09:01.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:01.808553 19217 rpc_server.cc:307] RPC server started. Bound to: 127.18.175.189:45157
20:09:01.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:01.808650 19290 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.18.175.189:45157 every 8 connection(s)
20:09:01.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:01.811194 19217 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster934885147846391950/master-1/data/info.pb
20:09:01.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:01.816063 19291 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
20:09:01.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:01.818771 19134 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskQoB_wm/build/tsan/bin/kudu as pid 19217
20:09:01.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:01.819269 19134 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster934885147846391950/master-1/wal/instance
20:09:01.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:01.822161 19134 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskQoB_wm/build/tsan/bin/kudu
20:09:01.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskQoB_wm/build/tsan/bin/kudu
20:09:01.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster934885147846391950/master-2/wal
20:09:01.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster934885147846391950/master-2/data
20:09:01.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:09:01.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:09:01.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:09:01.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:09:01.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:09:01.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:09:01.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster934885147846391950/master-2/logs
20:09:01.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster934885147846391950/master-2/data/info.pb
20:09:01.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:09:01.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:09:01.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:09:01.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:09:01.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:09:01.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:09:01.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:09:01.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:09:01.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:09:01.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:09:01.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:09:01.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:09:01.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.175.188:46095
20:09:01.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.175.188
20:09:01.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:09:01.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.175.148:37841
20:09:01.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:09:01.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:09:01.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:09:01.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:09:01.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.18.175.190:35215,127.18.175.189:45157,127.18.175.188:46095
20:09:01.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:09:01.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35579/jwks.json with env {}
20:09:01.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:01.830790 19291 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.18.175.190" port: 35215 } has no permanent_uuid. Determining permanent_uuid...
20:09:01.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:01.856247 19291 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.18.175.189" port: 45157 } has no permanent_uuid. Determining permanent_uuid...
20:09:01.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:01.873142 19291 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.18.175.188" port: 46095 } has no permanent_uuid. Determining permanent_uuid...
20:09:01.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:01.875389 19241 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.18.175.188:46095: connect: Connection refused (error 111)
20:09:01.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:01.884734 19291 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.18.175.188:46095: Network error: Client connection negotiation failed: client connection to 127.18.175.188:46095: connect: Connection refused (error 111)
20:09:01.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:01.934273 19291 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.175.188" port: 46095 } attempt: 1
20:09:01.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:01.938400 19291 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.18.175.188:46095: Network error: Client connection negotiation failed: client connection to 127.18.175.188:46095: connect: Connection refused (error 111)
20:09:02.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:02.006855 19291 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.175.188" port: 46095 } attempt: 2
20:09:02.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:02.010363 19291 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.18.175.188:46095: Network error: Client connection negotiation failed: client connection to 127.18.175.188:46095: connect: Connection refused (error 111)
20:09:02.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:02.101751 19291 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.175.188" port: 46095 } attempt: 3
20:09:02.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:02.105691 19291 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.18.175.188:46095: Network error: Client connection negotiation failed: client connection to 127.18.175.188:46095: connect: Connection refused (error 111)
20:09:02.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:02.142218 19293 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:09:02.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:02.142810 19293 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:09:02.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:02.143359 19293 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:09:02.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:02.174760 19293 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
20:09:02.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:02.175123 19293 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:09:02.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:02.175362 19293 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
20:09:02.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:02.175684 19293 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
20:09:02.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:02.175920 19293 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
20:09:02.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:02.176126 19293 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:35579/jwks.json
20:09:02.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:02.210587 19293 master_runner.cc:386] Master server non-default flags:
20:09:02.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:09:02.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.175.148:37841
20:09:02.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:09:02.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:09:02.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster934885147846391950/master-2/data
20:09:02.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster934885147846391950/master-2/wal
20:09:02.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:09:02.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.18.175.190:35215,127.18.175.189:45157,127.18.175.188:46095
20:09:02.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:09:02.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:09:02.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:09:02.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.175.188:46095
20:09:02.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:09:02.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:09:02.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:09:02.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35579/jwks.json
20:09:02.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:09:02.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:09:02.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster934885147846391950/master-2/data/info.pb
20:09:02.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.175.188
20:09:02.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:09:02.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:09:02.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:09:02.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:09:02.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:09:02.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:09:02.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster934885147846391950/master-2/logs
20:09:02.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:09:02.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:09:02.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:09:02.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:09:02.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
20:09:02.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
20:09:02.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:09:02.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 19:19:31 UTC on ccycloud.quasar-tlrbor.root.comops.site
20:09:02.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676273
20:09:02.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:09:02.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:02.211845 19293 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:09:02.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:02.213907 19293 file_cache.cc:493] Constructed file cache file cache with capacity 419430
20:09:02.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:02.232064 19304 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:09:02.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:02.249109 19291 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.175.188" port: 46095 } attempt: 4
20:09:02.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:02.252786 19291 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.18.175.188:46095: Network error: Client connection negotiation failed: client connection to 127.18.175.188:46095: connect: Connection refused (error 111)
20:09:02.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:02.386996 19215 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.175.189" port: 45157 } attempt: 7
20:09:02.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:02.423619 19215 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.18.175.188" port: 46095 } has no permanent_uuid. Determining permanent_uuid...
20:09:02.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:02.432173 19215 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.18.175.188:46095: Network error: Client connection negotiation failed: client connection to 127.18.175.188:46095: connect: Connection refused (error 111)
20:09:02.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:02.490710 19215 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.175.188" port: 46095 } attempt: 1
20:09:02.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:02.496444 19215 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.18.175.188:46095: Network error: Client connection negotiation failed: client connection to 127.18.175.188:46095: connect: Connection refused (error 111)
20:09:02.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:02.552196 19291 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.175.188" port: 46095 } attempt: 5
20:09:02.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:02.557947 19291 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.18.175.188:46095: Network error: Client connection negotiation failed: client connection to 127.18.175.188:46095: connect: Connection refused (error 111)
20:09:02.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:02.577978 19215 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.175.188" port: 46095 } attempt: 2
20:09:02.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:02.583854 19215 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.18.175.188:46095: Network error: Client connection negotiation failed: client connection to 127.18.175.188:46095: connect: Connection refused (error 111)
20:09:02.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:02.669425 19215 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.175.188" port: 46095 } attempt: 3
20:09:02.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:02.675704 19215 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.18.175.188:46095: Network error: Client connection negotiation failed: client connection to 127.18.175.188:46095: connect: Connection refused (error 111)
20:09:02.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:02.816357 19215 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.175.188" port: 46095 } attempt: 4
20:09:02.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:02.823764 19215 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.18.175.188:46095: Network error: Client connection negotiation failed: client connection to 127.18.175.188:46095: connect: Connection refused (error 111)
20:09:03.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:03.105544 19291 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.175.188" port: 46095 } attempt: 6
20:09:03.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:03.107355 19215 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.175.188" port: 46095 } attempt: 5
20:09:03.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:03.116956 19215 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.18.175.188:46095: Network error: Client connection negotiation failed: client connection to 127.18.175.188:46095: connect: Connection refused (error 111)
20:09:03.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:03.124344 19291 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.18.175.188:46095: Network error: Client connection negotiation failed: client connection to 127.18.175.188:46095: connect: Connection refused (error 111)
20:09:03.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:02.232146 19303 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:09:03.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:02.233394 19306 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:09:03.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:03.456182 19305 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1222 milliseconds
20:09:03.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:03.456305 19293 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:09:03.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:03.457654 19293 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:09:03.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:03.460809 19293 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:09:03.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:03.462278 19293 hybrid_clock.cc:648] HybridClock initialized: now 1790539743462234 us; error 65 us; skew 500 ppm
20:09:03.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:03.463213 19293 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:09:03.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:03.477643 19293 webserver.cc:460] Webserver started at http://127.18.175.188:35983/ using document root <none> and password file <none>
20:09:03.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:03.478713 19293 fs_manager.cc:359] Metadata directory not provided
20:09:03.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:03.479015 19293 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:09:03.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:03.479627 19293 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
20:09:03.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:03.487082 19293 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster934885147846391950/master-2/data/instance:
20:09:03.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f5d3e896e0e7449792b6dc09f1b655d8"
20:09:03.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 20:09:03 on dist-test-slave-6w59"
20:09:03.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:03.488662 19293 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster934885147846391950/master-2/wal/instance:
20:09:03.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f5d3e896e0e7449792b6dc09f1b655d8"
20:09:03.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 20:09:03 on dist-test-slave-6w59"
20:09:03.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:03.498742 19293 fs_manager.cc:691] Time spent creating directory manager: real 0.009s user 0.010s sys 0.001s
20:09:03.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:03.505203 19318 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:09:03.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:03.506294 19293 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.004s
20:09:03.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:03.506644 19293 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster934885147846391950/master-2/data,/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster934885147846391950/master-2/wal
20:09:03.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f5d3e896e0e7449792b6dc09f1b655d8"
20:09:03.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 20:09:03 on dist-test-slave-6w59"
20:09:03.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:03.507103 19293 fs_report.cc:383] FS layout report
20:09:03.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:09:03.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster934885147846391950/master-2/wal
20:09:03.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster934885147846391950/master-2/wal
20:09:03.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster934885147846391950/master-2/data/data
20:09:03.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:09:03.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:09:03.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:09:03.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:09:03.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:09:03.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:09:03.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:09:03.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:09:03.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:09:03.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:09:03.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:09:03.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:03.569655 19293 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
20:09:03.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:03.571306 19293 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:09:03.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:03.571870 19293 kserver.cc:163] Server-wide thread pool size limit: 3276
20:09:03.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:03.645457 19293 rpc_server.cc:307] RPC server started. Bound to: 127.18.175.188:46095
20:09:03.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:03.645557 19369 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.18.175.188:46095 every 8 connection(s)
20:09:03.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:03.648475 19293 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster934885147846391950/master-2/data/info.pb
20:09:03.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:03.654608 19370 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
20:09:03.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:03.655589 19134 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskQoB_wm/build/tsan/bin/kudu as pid 19293
20:09:03.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:03.656052 19134 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster934885147846391950/master-2/wal/instance
20:09:03.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:03.669472 19215 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.175.188" port: 46095 } attempt: 6
20:09:03.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:03.673070 19370 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.18.175.190" port: 35215 } has no permanent_uuid. Determining permanent_uuid...
20:09:03.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:03.699970 19370 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.18.175.189" port: 45157 } has no permanent_uuid. Determining permanent_uuid...
20:09:03.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:03.709022 19215 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P fd68912fb21c4af8803a4eea5cb9b2de: Bootstrap starting.
20:09:03.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:03.714124 19370 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.18.175.188" port: 46095 } has no permanent_uuid. Determining permanent_uuid...
20:09:03.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:03.716986 19215 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P fd68912fb21c4af8803a4eea5cb9b2de: Neither blocks nor log segments found. Creating new log.
20:09:03.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:03.719166 19215 log.cc:824] T 00000000000000000000000000000000 P fd68912fb21c4af8803a4eea5cb9b2de: Log is configured to *not* fsync() on all Append() calls
20:09:03.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:03.734078 19370 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f5d3e896e0e7449792b6dc09f1b655d8: Bootstrap starting.
20:09:03.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:03.742528 19370 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P f5d3e896e0e7449792b6dc09f1b655d8: Neither blocks nor log segments found. Creating new log.
20:09:03.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:03.744423 19370 log.cc:824] T 00000000000000000000000000000000 P f5d3e896e0e7449792b6dc09f1b655d8: Log is configured to *not* fsync() on all Append() calls
20:09:03.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:03.751070 19215 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P fd68912fb21c4af8803a4eea5cb9b2de: No bootstrap required, opened a new log
20:09:03.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:03.758606 19370 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f5d3e896e0e7449792b6dc09f1b655d8: No bootstrap required, opened a new log
20:09:03.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:03.784017 19215 raft_consensus.cc:348] T 00000000000000000000000000000000 P fd68912fb21c4af8803a4eea5cb9b2de [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fd68912fb21c4af8803a4eea5cb9b2de" member_type: VOTER last_known_addr { host: "127.18.175.190" port: 35215 } } peers { permanent_uuid: "ffc88e8c26024be18ba7bfb1d890fe67" member_type: VOTER last_known_addr { host: "127.18.175.189" port: 45157 } } peers { permanent_uuid: "f5d3e896e0e7449792b6dc09f1b655d8" member_type: VOTER last_known_addr { host: "127.18.175.188" port: 46095 } }
20:09:03.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:03.785133 19215 raft_consensus.cc:374] T 00000000000000000000000000000000 P fd68912fb21c4af8803a4eea5cb9b2de [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:09:03.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:03.785805 19215 raft_consensus.cc:729] T 00000000000000000000000000000000 P fd68912fb21c4af8803a4eea5cb9b2de [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: fd68912fb21c4af8803a4eea5cb9b2de, State: Initialized, Role: FOLLOWER
20:09:03.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:03.787001 19215 consensus_queue.cc:260] T 00000000000000000000000000000000 P fd68912fb21c4af8803a4eea5cb9b2de [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fd68912fb21c4af8803a4eea5cb9b2de" member_type: VOTER last_known_addr { host: "127.18.175.190" port: 35215 } } peers { permanent_uuid: "ffc88e8c26024be18ba7bfb1d890fe67" member_type: VOTER last_known_addr { host: "127.18.175.189" port: 45157 } } peers { permanent_uuid: "f5d3e896e0e7449792b6dc09f1b655d8" member_type: VOTER last_known_addr { host: "127.18.175.188" port: 46095 } }
20:09:03.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:03.789194 19370 raft_consensus.cc:348] T 00000000000000000000000000000000 P f5d3e896e0e7449792b6dc09f1b655d8 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fd68912fb21c4af8803a4eea5cb9b2de" member_type: VOTER last_known_addr { host: "127.18.175.190" port: 35215 } } peers { permanent_uuid: "ffc88e8c26024be18ba7bfb1d890fe67" member_type: VOTER last_known_addr { host: "127.18.175.189" port: 45157 } } peers { permanent_uuid: "f5d3e896e0e7449792b6dc09f1b655d8" member_type: VOTER last_known_addr { host: "127.18.175.188" port: 46095 } }
20:09:03.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:03.789856 19370 raft_consensus.cc:374] T 00000000000000000000000000000000 P f5d3e896e0e7449792b6dc09f1b655d8 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:09:03.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:03.790136 19370 raft_consensus.cc:729] T 00000000000000000000000000000000 P f5d3e896e0e7449792b6dc09f1b655d8 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f5d3e896e0e7449792b6dc09f1b655d8, State: Initialized, Role: FOLLOWER
20:09:03.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:03.790228 19382 sys_catalog.cc:455] T 00000000000000000000000000000000 P fd68912fb21c4af8803a4eea5cb9b2de [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fd68912fb21c4af8803a4eea5cb9b2de" member_type: VOTER last_known_addr { host: "127.18.175.190" port: 35215 } } peers { permanent_uuid: "ffc88e8c26024be18ba7bfb1d890fe67" member_type: VOTER last_known_addr { host: "127.18.175.189" port: 45157 } } peers { permanent_uuid: "f5d3e896e0e7449792b6dc09f1b655d8" member_type: VOTER last_known_addr { host: "127.18.175.188" port: 46095 } } }
20:09:03.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:03.791265 19382 sys_catalog.cc:458] T 00000000000000000000000000000000 P fd68912fb21c4af8803a4eea5cb9b2de [sys.catalog]: This master's current role is: FOLLOWER
20:09:03.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:03.791106 19370 consensus_queue.cc:260] T 00000000000000000000000000000000 P f5d3e896e0e7449792b6dc09f1b655d8 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fd68912fb21c4af8803a4eea5cb9b2de" member_type: VOTER last_known_addr { host: "127.18.175.190" port: 35215 } } peers { permanent_uuid: "ffc88e8c26024be18ba7bfb1d890fe67" member_type: VOTER last_known_addr { host: "127.18.175.189" port: 45157 } } peers { permanent_uuid: "f5d3e896e0e7449792b6dc09f1b655d8" member_type: VOTER last_known_addr { host: "127.18.175.188" port: 46095 } }
20:09:03.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:03.792014 19215 sys_catalog.cc:564] T 00000000000000000000000000000000 P fd68912fb21c4af8803a4eea5cb9b2de [sys.catalog]: configured and running, proceeding with master startup.
20:09:03.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:03.794168 19383 sys_catalog.cc:455] T 00000000000000000000000000000000 P f5d3e896e0e7449792b6dc09f1b655d8 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fd68912fb21c4af8803a4eea5cb9b2de" member_type: VOTER last_known_addr { host: "127.18.175.190" port: 35215 } } peers { permanent_uuid: "ffc88e8c26024be18ba7bfb1d890fe67" member_type: VOTER last_known_addr { host: "127.18.175.189" port: 45157 } } peers { permanent_uuid: "f5d3e896e0e7449792b6dc09f1b655d8" member_type: VOTER last_known_addr { host: "127.18.175.188" port: 46095 } } }
20:09:03.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:03.795171 19383 sys_catalog.cc:458] T 00000000000000000000000000000000 P f5d3e896e0e7449792b6dc09f1b655d8 [sys.catalog]: This master's current role is: FOLLOWER
20:09:03.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:03.795920 19370 sys_catalog.cc:564] T 00000000000000000000000000000000 P f5d3e896e0e7449792b6dc09f1b655d8 [sys.catalog]: configured and running, proceeding with master startup.
20:09:03.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:03.827008 19324 tablet.cc:2369] T 00000000000000000000000000000000 P f5d3e896e0e7449792b6dc09f1b655d8: Can't schedule compaction. Clean time has not been advanced past its initial value.
20:09:03.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:03.866421 19404 catalog_manager.cc:1558] T 00000000000000000000000000000000 P f5d3e896e0e7449792b6dc09f1b655d8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:09:03.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:03.866775 19404 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:09:03.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:03.827559 19293 master_runner.cc:186] Error getting master registration for 127.18.175.190:35215: OK, instance_id { permanent_uuid: "fd68912fb21c4af8803a4eea5cb9b2de" instance_seqno: 1790539740011862 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
20:09:03.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:03.878269 19293 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:09:03.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:03.884584 19405 catalog_manager.cc:1558] T 00000000000000000000000000000000 P fd68912fb21c4af8803a4eea5cb9b2de: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:09:03.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:03.885219 19405 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:09:03.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:03.844089 19146 master_runner.cc:186] Error getting master registration for 127.18.175.189:45157: OK, instance_id { permanent_uuid: "ffc88e8c26024be18ba7bfb1d890fe67" instance_seqno: 1790539741789228 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
20:09:03.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:03.902432 19146 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:09:03.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:03.971321 19168 tablet.cc:2369] T 00000000000000000000000000000000 P fd68912fb21c4af8803a4eea5cb9b2de: Can't schedule compaction. Clean time has not been advanced past its initial value.
20:09:04.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:04.004134 19383 raft_consensus.cc:482] T 00000000000000000000000000000000 P f5d3e896e0e7449792b6dc09f1b655d8 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:09:04.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:04.004733 19383 raft_consensus.cc:504] T 00000000000000000000000000000000 P f5d3e896e0e7449792b6dc09f1b655d8 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fd68912fb21c4af8803a4eea5cb9b2de" member_type: VOTER last_known_addr { host: "127.18.175.190" port: 35215 } } peers { permanent_uuid: "ffc88e8c26024be18ba7bfb1d890fe67" member_type: VOTER last_known_addr { host: "127.18.175.189" port: 45157 } } peers { permanent_uuid: "f5d3e896e0e7449792b6dc09f1b655d8" member_type: VOTER last_known_addr { host: "127.18.175.188" port: 46095 } }
20:09:04.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:04.008497 19383 leader_election.cc:288] T 00000000000000000000000000000000 P f5d3e896e0e7449792b6dc09f1b655d8 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers fd68912fb21c4af8803a4eea5cb9b2de (127.18.175.190:35215), ffc88e8c26024be18ba7bfb1d890fe67 (127.18.175.189:45157)
20:09:04.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:04.009124 19190 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "f5d3e896e0e7449792b6dc09f1b655d8" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "fd68912fb21c4af8803a4eea5cb9b2de" is_pre_election: true
20:09:04.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:04.010093 19190 raft_consensus.cc:2454] T 00000000000000000000000000000000 P fd68912fb21c4af8803a4eea5cb9b2de [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate f5d3e896e0e7449792b6dc09f1b655d8 in term 0.
20:09:04.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:04.011631 19321 leader_election.cc:302] T 00000000000000000000000000000000 P f5d3e896e0e7449792b6dc09f1b655d8 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: f5d3e896e0e7449792b6dc09f1b655d8, fd68912fb21c4af8803a4eea5cb9b2de; no voters:
20:09:04.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:04.012972 19383 raft_consensus.cc:2781] T 00000000000000000000000000000000 P f5d3e896e0e7449792b6dc09f1b655d8 [term 0 FOLLOWER]: Leader pre-election won for term 1
20:09:04.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:04.013338 19383 raft_consensus.cc:482] T 00000000000000000000000000000000 P f5d3e896e0e7449792b6dc09f1b655d8 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
20:09:04.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:04.013665 19383 raft_consensus.cc:3037] T 00000000000000000000000000000000 P f5d3e896e0e7449792b6dc09f1b655d8 [term 0 FOLLOWER]: Advancing to term 1
20:09:04.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:04.022027 19383 raft_consensus.cc:504] T 00000000000000000000000000000000 P f5d3e896e0e7449792b6dc09f1b655d8 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fd68912fb21c4af8803a4eea5cb9b2de" member_type: VOTER last_known_addr { host: "127.18.175.190" port: 35215 } } peers { permanent_uuid: "ffc88e8c26024be18ba7bfb1d890fe67" member_type: VOTER last_known_addr { host: "127.18.175.189" port: 45157 } } peers { permanent_uuid: "f5d3e896e0e7449792b6dc09f1b655d8" member_type: VOTER last_known_addr { host: "127.18.175.188" port: 46095 } }
20:09:04.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:04.032276 19383 leader_election.cc:288] T 00000000000000000000000000000000 P f5d3e896e0e7449792b6dc09f1b655d8 [CANDIDATE]: Term 1 election: Requested vote from peers fd68912fb21c4af8803a4eea5cb9b2de (127.18.175.190:35215), ffc88e8c26024be18ba7bfb1d890fe67 (127.18.175.189:45157)
20:09:04.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:04.033160 19190 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "f5d3e896e0e7449792b6dc09f1b655d8" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "fd68912fb21c4af8803a4eea5cb9b2de"
20:09:04.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:04.033777 19190 raft_consensus.cc:3037] T 00000000000000000000000000000000 P fd68912fb21c4af8803a4eea5cb9b2de [term 0 FOLLOWER]: Advancing to term 1
20:09:04.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:04.051461 19190 raft_consensus.cc:2454] T 00000000000000000000000000000000 P fd68912fb21c4af8803a4eea5cb9b2de [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate f5d3e896e0e7449792b6dc09f1b655d8 in term 1.
20:09:04.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:04.053715 19321 leader_election.cc:302] T 00000000000000000000000000000000 P f5d3e896e0e7449792b6dc09f1b655d8 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: f5d3e896e0e7449792b6dc09f1b655d8, fd68912fb21c4af8803a4eea5cb9b2de; no voters:
20:09:04.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:04.060496 19383 raft_consensus.cc:2781] T 00000000000000000000000000000000 P f5d3e896e0e7449792b6dc09f1b655d8 [term 1 FOLLOWER]: Leader election won for term 1
20:09:04.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:04.077091 19383 raft_consensus.cc:686] T 00000000000000000000000000000000 P f5d3e896e0e7449792b6dc09f1b655d8 [term 1 LEADER]: Becoming Leader. State: Replica: f5d3e896e0e7449792b6dc09f1b655d8, State: Running, Role: LEADER
20:09:04.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:04.080163 19383 consensus_queue.cc:237] T 00000000000000000000000000000000 P f5d3e896e0e7449792b6dc09f1b655d8 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fd68912fb21c4af8803a4eea5cb9b2de" member_type: VOTER last_known_addr { host: "127.18.175.190" port: 35215 } } peers { permanent_uuid: "ffc88e8c26024be18ba7bfb1d890fe67" member_type: VOTER last_known_addr { host: "127.18.175.189" port: 45157 } } peers { permanent_uuid: "f5d3e896e0e7449792b6dc09f1b655d8" member_type: VOTER last_known_addr { host: "127.18.175.188" port: 46095 } }
20:09:04.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:04.103613 19408 sys_catalog.cc:455] T 00000000000000000000000000000000 P f5d3e896e0e7449792b6dc09f1b655d8 [sys.catalog]: SysCatalogTable state changed. Reason: New leader f5d3e896e0e7449792b6dc09f1b655d8. Latest consensus state: current_term: 1 leader_uuid: "f5d3e896e0e7449792b6dc09f1b655d8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fd68912fb21c4af8803a4eea5cb9b2de" member_type: VOTER last_known_addr { host: "127.18.175.190" port: 35215 } } peers { permanent_uuid: "ffc88e8c26024be18ba7bfb1d890fe67" member_type: VOTER last_known_addr { host: "127.18.175.189" port: 45157 } } peers { permanent_uuid: "f5d3e896e0e7449792b6dc09f1b655d8" member_type: VOTER last_known_addr { host: "127.18.175.188" port: 46095 } } }
20:09:04.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:04.106122 19408 sys_catalog.cc:458] T 00000000000000000000000000000000 P f5d3e896e0e7449792b6dc09f1b655d8 [sys.catalog]: This master's current role is: LEADER
20:09:04.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:04.114004 19410 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
20:09:04.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:04.121922 19410 catalog_manager.cc:1484] Initializing Kudu cluster ID...
20:09:04.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:04.152433 19190 raft_consensus.cc:1261] T 00000000000000000000000000000000 P fd68912fb21c4af8803a4eea5cb9b2de [term 1 FOLLOWER]: Refusing update from remote peer f5d3e896e0e7449792b6dc09f1b655d8: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
20:09:04.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:04.159060 19408 consensus_queue.cc:1035] T 00000000000000000000000000000000 P f5d3e896e0e7449792b6dc09f1b655d8 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fd68912fb21c4af8803a4eea5cb9b2de" member_type: VOTER last_known_addr { host: "127.18.175.190" port: 35215 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
20:09:04.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:04.185710 19291 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.175.188" port: 46095 } attempt: 7
20:09:04.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:04.201323 19382 sys_catalog.cc:455] T 00000000000000000000000000000000 P fd68912fb21c4af8803a4eea5cb9b2de [sys.catalog]: SysCatalogTable state changed. Reason: New leader f5d3e896e0e7449792b6dc09f1b655d8. Latest consensus state: current_term: 1 leader_uuid: "f5d3e896e0e7449792b6dc09f1b655d8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fd68912fb21c4af8803a4eea5cb9b2de" member_type: VOTER last_known_addr { host: "127.18.175.190" port: 35215 } } peers { permanent_uuid: "ffc88e8c26024be18ba7bfb1d890fe67" member_type: VOTER last_known_addr { host: "127.18.175.189" port: 45157 } } peers { permanent_uuid: "f5d3e896e0e7449792b6dc09f1b655d8" member_type: VOTER last_known_addr { host: "127.18.175.188" port: 46095 } } }
20:09:04.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:04.207465 19382 sys_catalog.cc:458] T 00000000000000000000000000000000 P fd68912fb21c4af8803a4eea5cb9b2de [sys.catalog]: This master's current role is: FOLLOWER
20:09:04.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:04.219983 19411 mvcc.cc:205] Tried to move back new op lower bound from 7334050792025169920 to 7334050791785205760. Current Snapshot: MvccSnapshot[applied={T|T < 7334050792025169920}]
20:09:04.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:04.232435 19382 sys_catalog.cc:455] T 00000000000000000000000000000000 P fd68912fb21c4af8803a4eea5cb9b2de [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "f5d3e896e0e7449792b6dc09f1b655d8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fd68912fb21c4af8803a4eea5cb9b2de" member_type: VOTER last_known_addr { host: "127.18.175.190" port: 35215 } } peers { permanent_uuid: "ffc88e8c26024be18ba7bfb1d890fe67" member_type: VOTER last_known_addr { host: "127.18.175.189" port: 45157 } } peers { permanent_uuid: "f5d3e896e0e7449792b6dc09f1b655d8" member_type: VOTER last_known_addr { host: "127.18.175.188" port: 46095 } } }
20:09:04.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:04.233561 19382 sys_catalog.cc:458] T 00000000000000000000000000000000 P fd68912fb21c4af8803a4eea5cb9b2de [sys.catalog]: This master's current role is: FOLLOWER
20:09:04.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:04.240688 19383 sys_catalog.cc:455] T 00000000000000000000000000000000 P f5d3e896e0e7449792b6dc09f1b655d8 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "f5d3e896e0e7449792b6dc09f1b655d8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fd68912fb21c4af8803a4eea5cb9b2de" member_type: VOTER last_known_addr { host: "127.18.175.190" port: 35215 } } peers { permanent_uuid: "ffc88e8c26024be18ba7bfb1d890fe67" member_type: VOTER last_known_addr { host: "127.18.175.189" port: 45157 } } peers { permanent_uuid: "f5d3e896e0e7449792b6dc09f1b655d8" member_type: VOTER last_known_addr { host: "127.18.175.188" port: 46095 } } }
20:09:04.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:04.241493 19383 sys_catalog.cc:458] T 00000000000000000000000000000000 P f5d3e896e0e7449792b6dc09f1b655d8 [sys.catalog]: This master's current role is: LEADER
20:09:04.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:04.254340 19410 catalog_manager.cc:1347] Generated new cluster ID: b31a2424508f4a6e83a95602f2f0de96
20:09:04.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:04.254645 19410 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
20:09:04.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:04.276127 19291 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ffc88e8c26024be18ba7bfb1d890fe67: Bootstrap starting.
20:09:04.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:04.290131 19291 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P ffc88e8c26024be18ba7bfb1d890fe67: Neither blocks nor log segments found. Creating new log.
20:09:04.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:04.293607 19291 log.cc:824] T 00000000000000000000000000000000 P ffc88e8c26024be18ba7bfb1d890fe67: Log is configured to *not* fsync() on all Append() calls
20:09:04.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:04.312966 19291 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ffc88e8c26024be18ba7bfb1d890fe67: No bootstrap required, opened a new log
20:09:04.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:04.323030 19410 catalog_manager.cc:1370] Generated new certificate authority record
20:09:04.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:04.327595 19410 catalog_manager.cc:1504] Loading token signing keys...
20:09:04.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:04.382019 19410 catalog_manager.cc:5868] T 00000000000000000000000000000000 P f5d3e896e0e7449792b6dc09f1b655d8: Generated new TSK 0
20:09:04.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:04.384668 19410 catalog_manager.cc:1514] Initializing in-progress tserver states...
20:09:04.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:04.392279 19291 raft_consensus.cc:348] T 00000000000000000000000000000000 P ffc88e8c26024be18ba7bfb1d890fe67 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fd68912fb21c4af8803a4eea5cb9b2de" member_type: VOTER last_known_addr { host: "127.18.175.190" port: 35215 } } peers { permanent_uuid: "ffc88e8c26024be18ba7bfb1d890fe67" member_type: VOTER last_known_addr { host: "127.18.175.189" port: 45157 } } peers { permanent_uuid: "f5d3e896e0e7449792b6dc09f1b655d8" member_type: VOTER last_known_addr { host: "127.18.175.188" port: 46095 } }
20:09:04.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:04.394560 19291 raft_consensus.cc:374] T 00000000000000000000000000000000 P ffc88e8c26024be18ba7bfb1d890fe67 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:09:04.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:04.395360 19291 raft_consensus.cc:729] T 00000000000000000000000000000000 P ffc88e8c26024be18ba7bfb1d890fe67 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ffc88e8c26024be18ba7bfb1d890fe67, State: Initialized, Role: FOLLOWER
20:09:04.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:04.398196 19291 consensus_queue.cc:260] T 00000000000000000000000000000000 P ffc88e8c26024be18ba7bfb1d890fe67 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fd68912fb21c4af8803a4eea5cb9b2de" member_type: VOTER last_known_addr { host: "127.18.175.190" port: 35215 } } peers { permanent_uuid: "ffc88e8c26024be18ba7bfb1d890fe67" member_type: VOTER last_known_addr { host: "127.18.175.189" port: 45157 } } peers { permanent_uuid: "f5d3e896e0e7449792b6dc09f1b655d8" member_type: VOTER last_known_addr { host: "127.18.175.188" port: 46095 } }
20:09:04.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:04.407225 19419 sys_catalog.cc:455] T 00000000000000000000000000000000 P ffc88e8c26024be18ba7bfb1d890fe67 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fd68912fb21c4af8803a4eea5cb9b2de" member_type: VOTER last_known_addr { host: "127.18.175.190" port: 35215 } } peers { permanent_uuid: "ffc88e8c26024be18ba7bfb1d890fe67" member_type: VOTER last_known_addr { host: "127.18.175.189" port: 45157 } } peers { permanent_uuid: "f5d3e896e0e7449792b6dc09f1b655d8" member_type: VOTER last_known_addr { host: "127.18.175.188" port: 46095 } } }
20:09:04.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:04.408056 19419 sys_catalog.cc:458] T 00000000000000000000000000000000 P ffc88e8c26024be18ba7bfb1d890fe67 [sys.catalog]: This master's current role is: FOLLOWER
20:09:04.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:04.410909 19291 sys_catalog.cc:564] T 00000000000000000000000000000000 P ffc88e8c26024be18ba7bfb1d890fe67 [sys.catalog]: configured and running, proceeding with master startup.
20:09:04.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:04.411103 19264 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "f5d3e896e0e7449792b6dc09f1b655d8" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ffc88e8c26024be18ba7bfb1d890fe67"
20:09:04.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:04.411937 19264 raft_consensus.cc:3037] T 00000000000000000000000000000000 P ffc88e8c26024be18ba7bfb1d890fe67 [term 0 FOLLOWER]: Advancing to term 1
20:09:04.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:04.415721 19265 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "f5d3e896e0e7449792b6dc09f1b655d8" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ffc88e8c26024be18ba7bfb1d890fe67" is_pre_election: true
20:09:04.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:04.422677 19264 raft_consensus.cc:2454] T 00000000000000000000000000000000 P ffc88e8c26024be18ba7bfb1d890fe67 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate f5d3e896e0e7449792b6dc09f1b655d8 in term 1.
20:09:04.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:04.424132 19263 raft_consensus.cc:1261] T 00000000000000000000000000000000 P ffc88e8c26024be18ba7bfb1d890fe67 [term 1 FOLLOWER]: Refusing update from remote peer f5d3e896e0e7449792b6dc09f1b655d8: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
20:09:04.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:04.440922 19134 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskQoB_wm/build/tsan/bin/kudu
20:09:04.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskQoB_wm/build/tsan/bin/kudu
20:09:04.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster934885147846391950/ts-0/wal
20:09:04.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster934885147846391950/ts-0/data
20:09:04.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:09:04.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:09:04.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:09:04.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:09:04.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:09:04.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:09:04.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster934885147846391950/ts-0/logs
20:09:04.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster934885147846391950/ts-0/data/info.pb
20:09:04.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:09:04.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:09:04.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:09:04.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:09:04.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:09:04.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:09:04.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:09:04.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:09:04.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
20:09:04.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:09:04.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.175.129:0
20:09:04.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.18.175.129
20:09:04.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.175.129
20:09:04.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:09:04.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.18.175.190:35215,127.18.175.189:45157,127.18.175.188:46095
20:09:04.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.175.148:37841
20:09:04.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:09:04.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:09:04.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
20:09:04.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:04.444078 19408 consensus_queue.cc:1035] T 00000000000000000000000000000000 P f5d3e896e0e7449792b6dc09f1b655d8 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ffc88e8c26024be18ba7bfb1d890fe67" member_type: VOTER last_known_addr { host: "127.18.175.189" port: 45157 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
20:09:04.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:04.506076 19419 sys_catalog.cc:455] T 00000000000000000000000000000000 P ffc88e8c26024be18ba7bfb1d890fe67 [sys.catalog]: SysCatalogTable state changed. Reason: New leader f5d3e896e0e7449792b6dc09f1b655d8. Latest consensus state: current_term: 1 leader_uuid: "f5d3e896e0e7449792b6dc09f1b655d8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fd68912fb21c4af8803a4eea5cb9b2de" member_type: VOTER last_known_addr { host: "127.18.175.190" port: 35215 } } peers { permanent_uuid: "ffc88e8c26024be18ba7bfb1d890fe67" member_type: VOTER last_known_addr { host: "127.18.175.189" port: 45157 } } peers { permanent_uuid: "f5d3e896e0e7449792b6dc09f1b655d8" member_type: VOTER last_known_addr { host: "127.18.175.188" port: 46095 } } }
20:09:04.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:04.507072 19419 sys_catalog.cc:458] T 00000000000000000000000000000000 P ffc88e8c26024be18ba7bfb1d890fe67 [sys.catalog]: This master's current role is: FOLLOWER
20:09:04.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:04.506587 19431 sys_catalog.cc:455] T 00000000000000000000000000000000 P ffc88e8c26024be18ba7bfb1d890fe67 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "f5d3e896e0e7449792b6dc09f1b655d8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fd68912fb21c4af8803a4eea5cb9b2de" member_type: VOTER last_known_addr { host: "127.18.175.190" port: 35215 } } peers { permanent_uuid: "ffc88e8c26024be18ba7bfb1d890fe67" member_type: VOTER last_known_addr { host: "127.18.175.189" port: 45157 } } peers { permanent_uuid: "f5d3e896e0e7449792b6dc09f1b655d8" member_type: VOTER last_known_addr { host: "127.18.175.188" port: 46095 } } }
20:09:04.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:04.507664 19431 sys_catalog.cc:458] T 00000000000000000000000000000000 P ffc88e8c26024be18ba7bfb1d890fe67 [sys.catalog]: This master's current role is: FOLLOWER
20:09:04.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:04.532196 19436 catalog_manager.cc:1259] Loaded cluster ID: b31a2424508f4a6e83a95602f2f0de96
20:09:04.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:04.532766 19436 catalog_manager.cc:1552] T 00000000000000000000000000000000 P ffc88e8c26024be18ba7bfb1d890fe67: loading cluster ID for follower catalog manager: success
20:09:04.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:04.540961 19436 catalog_manager.cc:1574] T 00000000000000000000000000000000 P ffc88e8c26024be18ba7bfb1d890fe67: acquiring CA information for follower catalog manager: success
20:09:04.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:04.546710 19436 catalog_manager.cc:1602] T 00000000000000000000000000000000 P ffc88e8c26024be18ba7bfb1d890fe67: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
20:09:04.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:04.866456 19408 sys_catalog.cc:455] T 00000000000000000000000000000000 P f5d3e896e0e7449792b6dc09f1b655d8 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "f5d3e896e0e7449792b6dc09f1b655d8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fd68912fb21c4af8803a4eea5cb9b2de" member_type: VOTER last_known_addr { host: "127.18.175.190" port: 35215 } } peers { permanent_uuid: "ffc88e8c26024be18ba7bfb1d890fe67" member_type: VOTER last_known_addr { host: "127.18.175.189" port: 45157 } } peers { permanent_uuid: "f5d3e896e0e7449792b6dc09f1b655d8" member_type: VOTER last_known_addr { host: "127.18.175.188" port: 46095 } } }
20:09:04.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:04.867218 19408 sys_catalog.cc:458] T 00000000000000000000000000000000 P f5d3e896e0e7449792b6dc09f1b655d8 [sys.catalog]: This master's current role is: LEADER
20:09:04.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:04.891258 19405 catalog_manager.cc:1259] Loaded cluster ID: b31a2424508f4a6e83a95602f2f0de96
20:09:04.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:04.891875 19405 catalog_manager.cc:1552] T 00000000000000000000000000000000 P fd68912fb21c4af8803a4eea5cb9b2de: loading cluster ID for follower catalog manager: success
20:09:04.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:04.902808 19405 catalog_manager.cc:1574] T 00000000000000000000000000000000 P fd68912fb21c4af8803a4eea5cb9b2de: acquiring CA information for follower catalog manager: success
20:09:04.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:04.915584 19405 catalog_manager.cc:1602] T 00000000000000000000000000000000 P fd68912fb21c4af8803a4eea5cb9b2de: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
20:09:05.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:05.064101 19425 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:09:05.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:05.065127 19425 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:09:05.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:05.066176 19425 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:09:05.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:05.173382 19425 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:09:05.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:05.175100 19425 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.18.175.129
20:09:05.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:05.238032 19425 tablet_server_runner.cc:78] Tablet server non-default flags:
20:09:05.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:09:05.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.175.148:37841
20:09:05.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:09:05.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:09:05.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster934885147846391950/ts-0/data
20:09:05.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster934885147846391950/ts-0/wal
20:09:05.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:09:05.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:09:05.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.175.129:0
20:09:05.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:09:05.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:09:05.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:09:05.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster934885147846391950/ts-0/data/info.pb
20:09:05.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.175.129
20:09:05.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:09:05.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.18.175.190:35215,127.18.175.189:45157,127.18.175.188:46095
20:09:05.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:09:05.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:09:05.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:09:05.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:09:05.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:09:05.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.18.175.129
20:09:05.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster934885147846391950/ts-0/logs
20:09:05.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:09:05.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:09:05.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:09:05.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
20:09:05.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
20:09:05.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
20:09:05.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:09:05.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 19:19:31 UTC on ccycloud.quasar-tlrbor.root.comops.site
20:09:05.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676273
20:09:05.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:09:05.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:05.242123 19425 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:09:05.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:05.245419 19425 file_cache.cc:493] Constructed file cache file cache with capacity 419430
20:09:05.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:05.292575 19444 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:09:06.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:06.027410 19287 debug-util.cc:398] Leaking SignalData structure 0x7b0800076a80 after lost signal to thread 19221
20:09:06.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:06.028272 19287 debug-util.cc:398] Leaking SignalData structure 0x7b08000ac320 after lost signal to thread 19290
20:09:07.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:06.667227 19442 debug-util.cc:398] Leaking SignalData structure 0x7b0800014860 after lost signal to thread 19425
20:09:07.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:07.428725 19425 thread.cc:636] GCE (cloud detector) Time spent creating pthread: real 2.139s user 0.511s sys 0.768s
20:09:07.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:07.429106 19425 thread.cc:602] GCE (cloud detector) Time spent starting thread: real 2.139s user 0.511s sys 0.768s
20:09:07.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:05.294688 19443 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:09:07.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:07.438598 19448 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:09:07.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:07.447983 19425 server_base.cc:1034] running on GCE node
20:09:07.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:07.449803 19425 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:09:07.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:07.459846 19425 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:09:07.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:07.461503 19425 hybrid_clock.cc:648] HybridClock initialized: now 1790539747461430 us; error 78 us; skew 500 ppm
20:09:07.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:07.462680 19425 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:09:07.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:07.483004 19425 webserver.cc:460] Webserver started at http://127.18.175.129:34661/ using document root <none> and password file <none>
20:09:07.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:07.484453 19425 fs_manager.cc:359] Metadata directory not provided
20:09:07.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:07.484894 19425 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:09:07.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:07.485572 19425 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
20:09:07.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:07.495857 19425 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster934885147846391950/ts-0/data/instance:
20:09:07.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f7ea7dca257e4e39b930cef2f0911f1a"
20:09:07.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 20:09:07 on dist-test-slave-6w59"
20:09:07.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:07.497876 19425 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster934885147846391950/ts-0/wal/instance:
20:09:07.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f7ea7dca257e4e39b930cef2f0911f1a"
20:09:07.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 20:09:07 on dist-test-slave-6w59"
20:09:07.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:07.523409 19425 fs_manager.cc:691] Time spent creating directory manager: real 0.021s user 0.004s sys 0.016s
20:09:07.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:07.536727 19453 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:09:07.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:07.538648 19425 fs_manager.cc:725] Time spent opening block manager: real 0.011s user 0.007s sys 0.002s
20:09:07.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:07.539676 19425 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster934885147846391950/ts-0/data,/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster934885147846391950/ts-0/wal
20:09:07.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f7ea7dca257e4e39b930cef2f0911f1a"
20:09:07.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 20:09:07 on dist-test-slave-6w59"
20:09:07.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:07.540366 19425 fs_report.cc:383] FS layout report
20:09:07.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:09:07.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster934885147846391950/ts-0/wal
20:09:07.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster934885147846391950/ts-0/wal
20:09:07.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster934885147846391950/ts-0/data/data
20:09:07.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:09:07.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:09:07.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:09:07.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:09:07.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:09:07.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:09:07.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:09:07.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:09:07.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:09:07.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:09:07.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:09:07.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:07.629735 19425 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
20:09:07.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:07.631924 19425 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:09:07.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:07.632675 19425 kserver.cc:163] Server-wide thread pool size limit: 3276
20:09:07.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:07.638281 19425 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
20:09:07.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:07.651031 19425 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
20:09:07.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:07.651510 19425 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.001s user 0.000s sys 0.000s
20:09:07.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:07.651948 19425 ts_tablet_manager.cc:610] Registered 0 tablets
20:09:07.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:07.652295 19425 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
20:09:08.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:08.124864 19425 rpc_server.cc:307] RPC server started. Bound to: 127.18.175.129:46527
20:09:08.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:08.124975 19566 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.18.175.129:46527 every 8 connection(s)
20:09:08.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:08.128063 19425 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster934885147846391950/ts-0/data/info.pb
20:09:08.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:08.131640 19134 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskQoB_wm/build/tsan/bin/kudu as pid 19425
20:09:08.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:08.132128 19134 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster934885147846391950/ts-0/wal/instance
20:09:08.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:08.155431 19134 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskQoB_wm/build/tsan/bin/kudu
20:09:08.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskQoB_wm/build/tsan/bin/kudu
20:09:08.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster934885147846391950/ts-1/wal
20:09:08.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster934885147846391950/ts-1/data
20:09:08.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:09:08.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:09:08.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:09:08.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:09:08.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:09:08.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:09:08.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster934885147846391950/ts-1/logs
20:09:08.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster934885147846391950/ts-1/data/info.pb
20:09:08.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:09:08.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:09:08.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:09:08.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:09:08.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:09:08.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:09:08.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:09:08.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:09:08.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
20:09:08.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:09:08.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.175.130:0
20:09:08.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.18.175.130
20:09:08.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.175.130
20:09:08.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:09:08.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.18.175.190:35215,127.18.175.189:45157,127.18.175.188:46095
20:09:08.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.175.148:37841
20:09:08.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:09:08.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:09:08.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
20:09:08.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:08.201488 19567 heartbeater.cc:347] Connected to a master server at 127.18.175.188:46095
20:09:08.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:08.202009 19567 heartbeater.cc:464] Registering TS with master...
20:09:08.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:08.203267 19567 heartbeater.cc:511] Master 127.18.175.188:46095 requested a full tablet report, sending...
20:09:08.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:08.206306 19335 ts_manager.cc:194] Registered new tserver with Master: f7ea7dca257e4e39b930cef2f0911f1a (127.18.175.129:46527)
20:09:08.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:08.206538 19568 heartbeater.cc:347] Connected to a master server at 127.18.175.189:45157
20:09:08.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:08.206956 19568 heartbeater.cc:464] Registering TS with master...
20:09:08.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:08.207916 19568 heartbeater.cc:511] Master 127.18.175.189:45157 requested a full tablet report, sending...
20:09:08.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:08.209568 19335 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.18.175.129:44027
20:09:08.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:08.211381 19255 ts_manager.cc:194] Registered new tserver with Master: f7ea7dca257e4e39b930cef2f0911f1a (127.18.175.129:46527)
20:09:08.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:08.221765 19572 heartbeater.cc:347] Connected to a master server at 127.18.175.190:35215
20:09:08.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:08.222162 19572 heartbeater.cc:464] Registering TS with master...
20:09:08.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:08.222990 19572 heartbeater.cc:511] Master 127.18.175.190:35215 requested a full tablet report, sending...
20:09:08.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:08.225991 19179 ts_manager.cc:194] Registered new tserver with Master: f7ea7dca257e4e39b930cef2f0911f1a (127.18.175.129:46527)
20:09:08.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:08.503887 19571 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:09:08.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:08.504534 19571 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:09:08.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:08.505066 19571 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:09:08.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:08.536365 19571 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:09:08.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:08.537441 19571 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.18.175.130
20:09:08.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:08.572800 19571 tablet_server_runner.cc:78] Tablet server non-default flags:
20:09:08.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:09:08.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.175.148:37841
20:09:08.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:09:08.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:09:08.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster934885147846391950/ts-1/data
20:09:08.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster934885147846391950/ts-1/wal
20:09:08.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:09:08.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:09:08.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.175.130:0
20:09:08.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:09:08.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:09:08.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:09:08.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster934885147846391950/ts-1/data/info.pb
20:09:08.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.175.130
20:09:08.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:09:08.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.18.175.190:35215,127.18.175.189:45157,127.18.175.188:46095
20:09:08.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:09:08.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:09:08.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:09:08.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:09:08.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:09:08.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.18.175.130
20:09:08.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster934885147846391950/ts-1/logs
20:09:08.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:09:08.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:09:08.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:09:08.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
20:09:08.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
20:09:08.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
20:09:08.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:09:08.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 19:19:31 UTC on ccycloud.quasar-tlrbor.root.comops.site
20:09:08.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676273
20:09:08.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:09:08.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:08.574325 19571 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:09:08.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:08.576409 19571 file_cache.cc:493] Constructed file cache file cache with capacity 419430
20:09:08.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:08.603765 19586 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:09:09.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:09.214499 19567 heartbeater.cc:503] Master 127.18.175.188:46095 was elected leader, sending a full tablet report...
20:09:10.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:09.997188 19583 debug-util.cc:398] Leaking SignalData structure 0x7b0800014860 after lost signal to thread 19571
20:09:10.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:10.111955 19583 kernel_stack_watchdog.cc:198] Thread 19571 stuck at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/thread.cc:637 for 389ms:
20:09:10.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
20:09:10.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
20:09:10.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
20:09:10.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
20:09:10.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:08.603792 19585 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:09:10.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:10.113237 19571 thread.cc:636] OpenStack (cloud detector) Time spent creating pthread: real 1.508s user 0.001s sys 0.001s
20:09:10.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:10.113581 19571 thread.cc:602] OpenStack (cloud detector) Time spent starting thread: real 1.509s user 0.001s sys 0.001s
20:09:10.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:10.119498 19588 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:09:10.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:10.119747 19571 server_base.cc:1034] running on GCE node
20:09:10.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:10.120992 19571 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:09:10.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:10.124292 19571 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:09:10.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:10.125661 19571 hybrid_clock.cc:648] HybridClock initialized: now 1790539750125618 us; error 48 us; skew 500 ppm
20:09:10.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:10.126607 19571 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:09:10.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:10.137459 19571 webserver.cc:460] Webserver started at http://127.18.175.130:37779/ using document root <none> and password file <none>
20:09:10.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:10.138499 19571 fs_manager.cc:359] Metadata directory not provided
20:09:10.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:10.138731 19571 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:09:10.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:10.139252 19571 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
20:09:10.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:10.144042 19571 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster934885147846391950/ts-1/data/instance:
20:09:10.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a127da76d4e04217b608deb2a360fe8e"
20:09:10.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 20:09:10 on dist-test-slave-6w59"
20:09:10.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:10.145174 19571 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster934885147846391950/ts-1/wal/instance:
20:09:10.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a127da76d4e04217b608deb2a360fe8e"
20:09:10.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 20:09:10 on dist-test-slave-6w59"
20:09:10.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:10.153035 19571 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.005s sys 0.004s
20:09:10.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:10.158687 19595 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:09:10.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:10.159886 19571 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
20:09:10.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:10.160236 19571 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster934885147846391950/ts-1/data,/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster934885147846391950/ts-1/wal
20:09:10.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a127da76d4e04217b608deb2a360fe8e"
20:09:10.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 20:09:10 on dist-test-slave-6w59"
20:09:10.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:10.160630 19571 fs_report.cc:383] FS layout report
20:09:10.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:09:10.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster934885147846391950/ts-1/wal
20:09:10.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster934885147846391950/ts-1/wal
20:09:10.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster934885147846391950/ts-1/data/data
20:09:10.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:09:10.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:09:10.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:09:10.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:09:10.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:09:10.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:09:10.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:09:10.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:09:10.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:09:10.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:09:10.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:09:10.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:10.208442 19571 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
20:09:10.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:10.209906 19571 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:09:10.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:10.210381 19571 kserver.cc:163] Server-wide thread pool size limit: 3276
20:09:10.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:10.213017 19571 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
20:09:10.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:10.216969 19571 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
20:09:10.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:10.217197 19571 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
20:09:10.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:10.217530 19571 ts_tablet_manager.cc:610] Registered 0 tablets
20:09:10.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:10.217726 19571 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
20:09:10.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:10.362850 19571 rpc_server.cc:307] RPC server started. Bound to: 127.18.175.130:41001
20:09:10.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:10.363025 19708 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.18.175.130:41001 every 8 connection(s)
20:09:10.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:10.365516 19571 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster934885147846391950/ts-1/data/info.pb
20:09:10.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:10.373891 19134 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskQoB_wm/build/tsan/bin/kudu as pid 19571
20:09:10.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:10.374747 19134 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster934885147846391950/ts-1/wal/instance
20:09:10.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:10.393980 19134 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskQoB_wm/build/tsan/bin/kudu
20:09:10.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskQoB_wm/build/tsan/bin/kudu
20:09:10.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster934885147846391950/ts-2/wal
20:09:10.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster934885147846391950/ts-2/data
20:09:10.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:09:10.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:09:10.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:09:10.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:09:10.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:09:10.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:09:10.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster934885147846391950/ts-2/logs
20:09:10.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster934885147846391950/ts-2/data/info.pb
20:09:10.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:09:10.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:09:10.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:09:10.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:09:10.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:09:10.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:09:10.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:09:10.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:09:10.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
20:09:10.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:09:10.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.175.131:0
20:09:10.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.18.175.131
20:09:10.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.175.131
20:09:10.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:09:10.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.18.175.190:35215,127.18.175.189:45157,127.18.175.188:46095
20:09:10.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.175.148:37841
20:09:10.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:09:10.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:09:10.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
20:09:10.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:10.409209 19709 heartbeater.cc:347] Connected to a master server at 127.18.175.188:46095
20:09:10.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:10.409709 19709 heartbeater.cc:464] Registering TS with master...
20:09:10.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:10.409767 19711 heartbeater.cc:347] Connected to a master server at 127.18.175.190:35215
20:09:10.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:10.410153 19711 heartbeater.cc:464] Registering TS with master...
20:09:10.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:10.411120 19709 heartbeater.cc:511] Master 127.18.175.188:46095 requested a full tablet report, sending...
20:09:10.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:10.411470 19711 heartbeater.cc:511] Master 127.18.175.190:35215 requested a full tablet report, sending...
20:09:10.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:10.413928 19335 ts_manager.cc:194] Registered new tserver with Master: a127da76d4e04217b608deb2a360fe8e (127.18.175.130:41001)
20:09:10.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:10.415325 19179 ts_manager.cc:194] Registered new tserver with Master: a127da76d4e04217b608deb2a360fe8e (127.18.175.130:41001)
20:09:10.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:10.415724 19335 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.18.175.130:50737
20:09:10.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:10.417466 19710 heartbeater.cc:347] Connected to a master server at 127.18.175.189:45157
20:09:10.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:10.417778 19710 heartbeater.cc:464] Registering TS with master...
20:09:10.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:10.418457 19710 heartbeater.cc:511] Master 127.18.175.189:45157 requested a full tablet report, sending...
20:09:10.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:10.421031 19255 ts_manager.cc:194] Registered new tserver with Master: a127da76d4e04217b608deb2a360fe8e (127.18.175.130:41001)
20:09:10.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:10.726096 19719 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:09:10.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:10.726742 19719 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:09:10.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:10.727263 19719 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:09:10.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:10.758544 19719 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:09:10.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:10.759415 19719 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.18.175.131
20:09:10.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:10.793416 19719 tablet_server_runner.cc:78] Tablet server non-default flags:
20:09:10.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:09:10.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.175.148:37841
20:09:10.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:09:10.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:09:10.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster934885147846391950/ts-2/data
20:09:10.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster934885147846391950/ts-2/wal
20:09:10.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:09:10.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:09:10.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.175.131:0
20:09:10.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:09:10.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:09:10.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:09:10.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster934885147846391950/ts-2/data/info.pb
20:09:10.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.175.131
20:09:10.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:09:10.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.18.175.190:35215,127.18.175.189:45157,127.18.175.188:46095
20:09:10.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:09:10.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:09:10.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:09:10.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:09:10.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:09:10.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.18.175.131
20:09:10.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster934885147846391950/ts-2/logs
20:09:10.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:09:10.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:09:10.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:09:10.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
20:09:10.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
20:09:10.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
20:09:10.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:09:10.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 19:19:31 UTC on ccycloud.quasar-tlrbor.root.comops.site
20:09:10.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676273
20:09:10.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:09:10.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:10.794540 19719 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:09:10.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:10.796159 19719 file_cache.cc:493] Constructed file cache file cache with capacity 419430
20:09:10.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:10.815547 19725 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:09:11.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:11.422835 19709 heartbeater.cc:503] Master 127.18.175.188:46095 was elected leader, sending a full tablet report...
20:09:12.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:10.815586 19726 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:09:12.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:10.817204 19719 server_base.cc:1034] running on GCE node
20:09:12.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:10.817068 19728 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:09:12.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:12.026181 19719 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:09:12.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:12.028688 19719 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:09:12.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:12.030030 19719 hybrid_clock.cc:648] HybridClock initialized: now 1790539752029999 us; error 51 us; skew 500 ppm
20:09:12.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:12.030864 19719 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:09:12.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:12.038365 19719 webserver.cc:460] Webserver started at http://127.18.175.131:46731/ using document root <none> and password file <none>
20:09:12.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:12.039383 19719 fs_manager.cc:359] Metadata directory not provided
20:09:12.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:12.039613 19719 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:09:12.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:12.040210 19719 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
20:09:12.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:12.045086 19719 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster934885147846391950/ts-2/data/instance:
20:09:12.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b523dcd2eaf142eca3e3e4d9c9abb082"
20:09:12.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 20:09:12 on dist-test-slave-6w59"
20:09:12.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:12.046095 19719 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster934885147846391950/ts-2/wal/instance:
20:09:12.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b523dcd2eaf142eca3e3e4d9c9abb082"
20:09:12.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 20:09:12 on dist-test-slave-6w59"
20:09:12.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:12.053531 19719 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.005s sys 0.004s
20:09:12.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:12.059324 19735 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:09:12.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:12.060286 19719 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.002s sys 0.002s
20:09:12.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:12.060637 19719 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster934885147846391950/ts-2/data,/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster934885147846391950/ts-2/wal
20:09:12.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b523dcd2eaf142eca3e3e4d9c9abb082"
20:09:12.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 20:09:12 on dist-test-slave-6w59"
20:09:12.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:12.060992 19719 fs_report.cc:383] FS layout report
20:09:12.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:09:12.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster934885147846391950/ts-2/wal
20:09:12.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster934885147846391950/ts-2/wal
20:09:12.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster934885147846391950/ts-2/data/data
20:09:12.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:09:12.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:09:12.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:09:12.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:09:12.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:09:12.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:09:12.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:09:12.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:09:12.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:09:12.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:09:12.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:09:12.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:12.096637 19719 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
20:09:12.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:12.098219 19719 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:09:12.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:12.098724 19719 kserver.cc:163] Server-wide thread pool size limit: 3276
20:09:12.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:12.101287 19719 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
20:09:12.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:12.105196 19719 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
20:09:12.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:12.105422 19719 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
20:09:12.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:12.105759 19719 ts_tablet_manager.cc:610] Registered 0 tablets
20:09:12.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:12.105949 19719 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
20:09:12.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:12.254019 19719 rpc_server.cc:307] RPC server started. Bound to: 127.18.175.131:39115
20:09:12.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:12.254189 19848 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.18.175.131:39115 every 8 connection(s)
20:09:12.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:12.256903 19719 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster934885147846391950/ts-2/data/info.pb
20:09:12.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:12.263849 19134 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskQoB_wm/build/tsan/bin/kudu as pid 19719
20:09:12.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:12.264393 19134 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster934885147846391950/ts-2/wal/instance
20:09:12.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:12.310096 19849 heartbeater.cc:347] Connected to a master server at 127.18.175.188:46095
20:09:12.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:12.310576 19849 heartbeater.cc:464] Registering TS with master...
20:09:12.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:12.311431 19851 heartbeater.cc:347] Connected to a master server at 127.18.175.190:35215
20:09:12.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:12.311673 19851 heartbeater.cc:464] Registering TS with master...
20:09:12.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:12.311810 19849 heartbeater.cc:511] Master 127.18.175.188:46095 requested a full tablet report, sending...
20:09:12.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:12.312634 19851 heartbeater.cc:511] Master 127.18.175.190:35215 requested a full tablet report, sending...
20:09:12.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:12.314707 19335 ts_manager.cc:194] Registered new tserver with Master: b523dcd2eaf142eca3e3e4d9c9abb082 (127.18.175.131:39115)
20:09:12.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:12.315675 19850 heartbeater.cc:347] Connected to a master server at 127.18.175.189:45157
20:09:12.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:12.316015 19850 heartbeater.cc:464] Registering TS with master...
20:09:12.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:12.316105 19179 ts_manager.cc:194] Registered new tserver with Master: b523dcd2eaf142eca3e3e4d9c9abb082 (127.18.175.131:39115)
20:09:12.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:12.316768 19850 heartbeater.cc:511] Master 127.18.175.189:45157 requested a full tablet report, sending...
20:09:12.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:12.317018 19335 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.18.175.131:50351
20:09:12.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:12.319298 19255 ts_manager.cc:194] Registered new tserver with Master: b523dcd2eaf142eca3e3e4d9c9abb082 (127.18.175.131:39115)
20:09:12.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:12.332993 19134 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
20:09:12.343 [DEBUG - main] (MiniKuduCluster.java:195) Response:
20:09:12.344 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
20:09:12.356 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.18.175.190"
port: 35215
}
bound_http_address {
host: "127.18.175.190"
port: 46121
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.18.175.189"
port: 45157
}
bound_http_address {
host: "127.18.175.189"
port: 33577
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.18.175.188"
port: 46095
}
bound_http_address {
host: "127.18.175.188"
port: 35983
}
}
}
20:09:12.362 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
20:09:12.365 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.18.175.129"
port: 46527
}
bound_http_address {
host: "127.18.175.129"
port: 34661
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.18.175.130"
port: 41001
}
bound_http_address {
host: "127.18.175.130"
port: 37779
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.18.175.131"
port: 39115
}
bound_http_address {
host: "127.18.175.131"
port: 46731
}
}
}
20:09:12.365 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
20:09:12.545 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
20:09:12.666 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskQoB_wm/test-tmp
20:09:12.667 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
20:09:12.667 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskQoB_wm/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
20:09:12.673 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035"
extra_master_flags: "--authn_token_validity_seconds=10"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
20:09:13.000 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:09:13.000 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
20:09:13.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:13.003808 19860 mini_oidc.cc:152] Starting JWKS server
20:09:13.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:13.011780 19860 webserver.cc:460] Webserver started at http://127.0.0.1:39755/ using document root <none> and password file <none>
20:09:13.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:13.012389 19860 mini_oidc.cc:189] Starting OIDC Discovery server
20:09:13.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:13.014452 19860 webserver.cc:460] Webserver started at http://127.0.0.1:34703/ using document root <none> and password file <none>
20:09:13.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
20:09:13.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/krb5kdc/principal' for realm 'KRBTEST.COM',
20:09:13.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
20:09:13.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 20:09:13 dist-test-slave-6w59 krb5kdc[19872](info): setting up network...
20:09:13.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
20:09:13.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 20:09:13 dist-test-slave-6w59 krb5kdc[19872](info): set up 2 sockets
20:09:13.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 20:09:13 dist-test-slave-6w59 krb5kdc[19872](info): commencing operation
20:09:13.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
20:09:13.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:13.321017 19849 heartbeater.cc:503] Master 127.18.175.188:46095 was elected leader, sending a full tablet report...
20:09:14.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:14.141047 19860 mini_kdc.cc:121] Time spent starting KDC: real 1.098s user 0.003s sys 0.018s
20:09:14.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
20:09:14.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
20:09:14.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
20:09:14.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
20:09:14.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
20:09:14.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
20:09:14.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
20:09:14.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
20:09:14.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
20:09:14.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
20:09:14.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/krb5kdc/test-user.keytab.
20:09:14.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/krb5kdc/test-user.keytab.
20:09:14.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 20:09:14 dist-test-slave-6w59 krb5kdc[19872](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790539754, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:09:14.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
20:09:14.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-27T20:09:14Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
20:09:14.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-27T20:09:14Z Disabled control of system clock
20:09:14.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.19.101.62@KRBTEST.COM; defaulting to no policy
20:09:14.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:09:14.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.19.101.62@KRBTEST.COM" created.
20:09:14.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:09:14.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.19.101.62 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/krb5kdc/kudu_127.19.101.62.keytab.
20:09:14.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.19.101.62 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/krb5kdc/kudu_127.19.101.62.keytab.
20:09:14.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:14.336016 19860 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskQoB_wm/build/tsan/bin/kudu
20:09:14.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskQoB_wm/build/tsan/bin/kudu
20:09:14.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/master-0/wal
20:09:14.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/master-0/data
20:09:14.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:09:14.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:09:14.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:09:14.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:09:14.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:09:14.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:09:14.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/master-0/logs
20:09:14.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/master-0/data/info.pb
20:09:14.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:09:14.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:09:14.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:09:14.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:09:14.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:09:14.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:09:14.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:09:14.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:09:14.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:09:14.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:09:14.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:09:14.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:09:14.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.101.62:46347
20:09:14.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.101.62
20:09:14.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:09:14.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/krb5kdc/kudu_127.19.101.62.keytab
20:09:14.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.19.101.62
20:09:14.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:09:14.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:09:14.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:09:14.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.101.20:37005
20:09:14.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:09:14.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:09:14.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:09:14.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:09:14.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.19.101.62:46347,127.19.101.61:33871,127.19.101.60:35091
20:09:14.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:09:14.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39755/jwks.json
20:09:14.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:09:14.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:14.673388 19885 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:09:14.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:14.674065 19885 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:09:14.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:14.674783 19885 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:09:14.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:14.707409 19885 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
20:09:14.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:14.707684 19885 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
20:09:14.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:14.708068 19885 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:09:14.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:14.708396 19885 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
20:09:14.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:14.708700 19885 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
20:09:14.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:14.709004 19885 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
20:09:14.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:14.709228 19885 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:39755/jwks.json
20:09:14.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:14.745972 19885 master_runner.cc:386] Master server non-default flags:
20:09:14.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:09:14.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.101.20:37005
20:09:14.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:09:14.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:09:14.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/master-0/data
20:09:14.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/master-0/wal
20:09:14.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
20:09:14.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:09:14.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.19.101.62:46347,127.19.101.61:33871,127.19.101.60:35091
20:09:14.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/krb5kdc/kudu_127.19.101.62.keytab
20:09:14.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.19.101.62
20:09:14.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:09:14.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:09:14.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:09:14.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.101.62:46347
20:09:14.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:09:14.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:09:14.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:09:14.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39755/jwks.json
20:09:14.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:09:14.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:09:14.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:09:14.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:09:14.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:09:14.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/master-0/data/info.pb
20:09:14.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.101.62
20:09:14.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:09:14.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:09:14.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:09:14.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:09:14.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:09:14.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:09:14.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/master-0/logs
20:09:14.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:09:14.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:09:14.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:09:14.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:09:14.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
20:09:14.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
20:09:14.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:09:14.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 19:19:31 UTC on ccycloud.quasar-tlrbor.root.comops.site
20:09:14.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676273
20:09:14.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:09:14.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:14.747529 19885 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:09:14.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:14.748695 19885 file_cache.cc:493] Constructed file cache file cache with capacity 419430
20:09:14.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:14.767683 19892 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:09:15.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:14.768687 19894 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:09:15.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:14.767678 19891 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:09:15.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:15.993573 19885 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:09:15.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:15.994021 19893 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
20:09:15.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:15.997973 19885 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:09:16.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:16.001067 19885 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:09:16.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:16.002629 19885 hybrid_clock.cc:648] HybridClock initialized: now 1790539756002566 us; error 66 us; skew 500 ppm
20:09:16.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:16.003587 19885 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:09:16.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 20:09:16 dist-test-slave-6w59 krb5kdc[19872](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790539756, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.101.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:09:16.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:16.014685 19885 init.cc:377] Logged in from keytab as kudu/127.19.101.62@KRBTEST.COM (short username kudu)
20:09:16.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:16.020375 19885 webserver.cc:460] Webserver started at http://127.19.101.62:43047/ using document root <none> and password file <none>
20:09:16.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:16.021318 19885 fs_manager.cc:359] Metadata directory not provided
20:09:16.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:16.021529 19885 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:09:16.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:16.021951 19885 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
20:09:16.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:16.027719 19885 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/master-0/data/instance:
20:09:16.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d2412d2667ad45169185672e68bf0e7c"
20:09:16.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 20:09:16 on dist-test-slave-6w59"
20:09:16.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:16.028879 19885 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/master-0/wal/instance:
20:09:16.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d2412d2667ad45169185672e68bf0e7c"
20:09:16.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 20:09:16 on dist-test-slave-6w59"
20:09:16.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:16.038910 19885 fs_manager.cc:691] Time spent creating directory manager: real 0.009s user 0.003s sys 0.008s
20:09:16.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:16.046240 19904 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:09:16.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:16.047598 19885 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
20:09:16.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:16.048008 19885 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/master-0/data,/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/master-0/wal
20:09:16.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d2412d2667ad45169185672e68bf0e7c"
20:09:16.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 20:09:16 on dist-test-slave-6w59"
20:09:16.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:16.048478 19885 fs_report.cc:383] FS layout report
20:09:16.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:09:16.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/master-0/wal
20:09:16.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/master-0/wal
20:09:16.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/master-0/data/data
20:09:16.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:09:16.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:09:16.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:09:16.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:09:16.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:09:16.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:09:16.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:09:16.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:09:16.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:09:16.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:09:16.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:09:16.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:16.096316 19885 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
20:09:16.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:16.102067 19885 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:09:16.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:16.102474 19885 kserver.cc:163] Server-wide thread pool size limit: 3276
20:09:16.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:16.176709 19885 rpc_server.cc:307] RPC server started. Bound to: 127.19.101.62:46347
20:09:16.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:16.176810 19955 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.19.101.62:46347 every 8 connection(s)
20:09:16.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:16.179358 19885 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/master-0/data/info.pb
20:09:16.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:16.184962 19956 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
20:09:16.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:16.186795 19860 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskQoB_wm/build/tsan/bin/kudu as pid 19885
20:09:16.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:16.187368 19860 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/master-0/wal/instance
20:09:16.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:16.200333 19956 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.19.101.62" port: 46347 } has no permanent_uuid. Determining permanent_uuid...
20:09:16.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.19.101.61@KRBTEST.COM; defaulting to no policy
20:09:16.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:09:16.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.19.101.61@KRBTEST.COM" created.
20:09:16.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 20:09:16 dist-test-slave-6w59 krb5kdc[19872](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790539756, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.101.62@KRBTEST.COM for kudu/127.19.101.62@KRBTEST.COM
20:09:16.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:09:16.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.19.101.61 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/krb5kdc/kudu_127.19.101.61.keytab.
20:09:16.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.19.101.61 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/krb5kdc/kudu_127.19.101.61.keytab.
20:09:16.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:16.237545 19860 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskQoB_wm/build/tsan/bin/kudu
20:09:16.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskQoB_wm/build/tsan/bin/kudu
20:09:16.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/master-1/wal
20:09:16.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/master-1/data
20:09:16.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:09:16.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:09:16.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:09:16.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:09:16.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:09:16.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:09:16.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/master-1/logs
20:09:16.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/master-1/data/info.pb
20:09:16.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:09:16.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:09:16.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:09:16.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:09:16.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:09:16.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:09:16.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:09:16.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:09:16.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:09:16.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:09:16.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:09:16.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:09:16.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.101.61:33871
20:09:16.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.101.61
20:09:16.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:09:16.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/krb5kdc/kudu_127.19.101.61.keytab
20:09:16.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.19.101.61
20:09:16.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:09:16.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:09:16.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:09:16.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.101.20:37005
20:09:16.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:09:16.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:09:16.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:09:16.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:09:16.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.19.101.62:46347,127.19.101.61:33871,127.19.101.60:35091
20:09:16.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:09:16.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39755/jwks.json
20:09:16.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:09:16.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:16.245659 19956 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.19.101.61" port: 33871 } has no permanent_uuid. Determining permanent_uuid...
20:09:16.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:16.248111 19908 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.19.101.61:33871: connect: Connection refused (error 111)
20:09:16.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:16.252070 19956 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.19.101.61:33871: Network error: Client connection negotiation failed: client connection to 127.19.101.61:33871: connect: Connection refused (error 111)
20:09:16.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:16.301569 19956 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.101.61" port: 33871 } attempt: 1
20:09:16.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:16.305182 19956 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.19.101.61:33871: Network error: Client connection negotiation failed: client connection to 127.19.101.61:33871: connect: Connection refused (error 111)
20:09:16.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:16.373625 19956 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.101.61" port: 33871 } attempt: 2
20:09:16.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:16.377665 19956 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.19.101.61:33871: Network error: Client connection negotiation failed: client connection to 127.19.101.61:33871: connect: Connection refused (error 111)
20:09:16.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:16.469026 19956 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.101.61" port: 33871 } attempt: 3
20:09:16.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:16.472517 19956 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.19.101.61:33871: Network error: Client connection negotiation failed: client connection to 127.19.101.61:33871: connect: Connection refused (error 111)
20:09:16.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:16.561180 19962 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:09:16.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:16.561725 19962 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:09:16.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:16.562283 19962 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:09:16.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:16.593685 19962 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
20:09:16.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:16.593907 19962 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
20:09:16.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:16.594157 19962 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:09:16.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:16.594349 19962 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
20:09:16.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:16.594543 19962 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
20:09:16.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:16.594724 19962 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
20:09:16.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:16.594859 19962 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:39755/jwks.json
20:09:16.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:16.615902 19956 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.101.61" port: 33871 } attempt: 4
20:09:16.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:16.619596 19956 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.19.101.61:33871: Network error: Client connection negotiation failed: client connection to 127.19.101.61:33871: connect: Connection refused (error 111)
20:09:16.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:16.634379 19962 master_runner.cc:386] Master server non-default flags:
20:09:16.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:09:16.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.101.20:37005
20:09:16.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:09:16.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:09:16.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/master-1/data
20:09:16.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/master-1/wal
20:09:16.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
20:09:16.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:09:16.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.19.101.62:46347,127.19.101.61:33871,127.19.101.60:35091
20:09:16.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/krb5kdc/kudu_127.19.101.61.keytab
20:09:16.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.19.101.61
20:09:16.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:09:16.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:09:16.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:09:16.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.101.61:33871
20:09:16.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:09:16.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:09:16.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:09:16.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39755/jwks.json
20:09:16.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:09:16.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:09:16.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:09:16.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:09:16.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:09:16.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/master-1/data/info.pb
20:09:16.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.101.61
20:09:16.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:09:16.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:09:16.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:09:16.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:09:16.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:09:16.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:09:16.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/master-1/logs
20:09:16.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:09:16.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:09:16.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:09:16.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:09:16.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
20:09:16.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
20:09:16.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:09:16.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 19:19:31 UTC on ccycloud.quasar-tlrbor.root.comops.site
20:09:16.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676273
20:09:16.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:09:16.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:16.635874 19962 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:09:16.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:16.637207 19962 file_cache.cc:493] Constructed file cache file cache with capacity 419430
20:09:16.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:16.655074 19970 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:09:16.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:16.918988 19956 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.101.61" port: 33871 } attempt: 5
20:09:16.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:16.923725 19956 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.19.101.61:33871: Network error: Client connection negotiation failed: client connection to 127.19.101.61:33871: connect: Connection refused (error 111)
20:09:17.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:17.471138 19956 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.101.61" port: 33871 } attempt: 6
20:09:17.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:17.477632 19956 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.19.101.61:33871: Network error: Client connection negotiation failed: client connection to 127.19.101.61:33871: connect: Connection refused (error 111)
20:09:17.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:16.655994 19969 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:09:17.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:16.656172 19972 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:09:17.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:16.657395 19962 server_base.cc:1034] running on GCE node
20:09:17.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:17.878278 19962 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:09:17.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:17.881130 19962 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:09:17.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:17.882515 19962 hybrid_clock.cc:648] HybridClock initialized: now 1790539757882477 us; error 49 us; skew 500 ppm
20:09:17.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:17.883394 19962 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:09:17.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 20:09:17 dist-test-slave-6w59 krb5kdc[19872](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790539757, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.101.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:09:17.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:17.889317 19962 init.cc:377] Logged in from keytab as kudu/127.19.101.61@KRBTEST.COM (short username kudu)
20:09:17.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:17.894027 19962 webserver.cc:460] Webserver started at http://127.19.101.61:36079/ using document root <none> and password file <none>
20:09:17.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:17.895035 19962 fs_manager.cc:359] Metadata directory not provided
20:09:17.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:17.895260 19962 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:09:17.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:17.895750 19962 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
20:09:17.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:17.900906 19962 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/master-1/data/instance:
20:09:17.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "21c7a3e6c1c54335822620013c20745f"
20:09:17.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 20:09:17 on dist-test-slave-6w59"
20:09:17.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:17.902015 19962 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/master-1/wal/instance:
20:09:17.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "21c7a3e6c1c54335822620013c20745f"
20:09:17.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 20:09:17 on dist-test-slave-6w59"
20:09:17.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:17.909672 19962 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.009s sys 0.000s
20:09:17.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:17.915588 19982 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:09:17.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:17.916541 19962 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.004s sys 0.000s
20:09:17.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:17.916898 19962 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/master-1/data,/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/master-1/wal
20:09:17.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "21c7a3e6c1c54335822620013c20745f"
20:09:17.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 20:09:17 on dist-test-slave-6w59"
20:09:17.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:17.917240 19962 fs_report.cc:383] FS layout report
20:09:17.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:09:17.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/master-1/wal
20:09:17.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/master-1/wal
20:09:17.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/master-1/data/data
20:09:17.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:09:17.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:09:17.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:09:17.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:09:17.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:09:17.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:09:17.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:09:17.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:09:17.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:09:17.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:09:17.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:09:17.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:17.976531 19962 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
20:09:17.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:17.982342 19962 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:09:17.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:17.982741 19962 kserver.cc:163] Server-wide thread pool size limit: 3276
20:09:18.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:18.055833 19962 rpc_server.cc:307] RPC server started. Bound to: 127.19.101.61:33871
20:09:18.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:18.055891 20033 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.19.101.61:33871 every 8 connection(s)
20:09:18.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:18.059369 19962 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/master-1/data/info.pb
20:09:18.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:18.060367 19860 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskQoB_wm/build/tsan/bin/kudu as pid 19962
20:09:18.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:18.060887 19860 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/master-1/wal/instance
20:09:18.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:18.066004 20034 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
20:09:18.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.19.101.60@KRBTEST.COM; defaulting to no policy
20:09:18.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:18.081303 20034 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.19.101.62" port: 46347 } has no permanent_uuid. Determining permanent_uuid...
20:09:18.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:09:18.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.19.101.60@KRBTEST.COM" created.
20:09:18.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 20:09:18 dist-test-slave-6w59 krb5kdc[19872](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790539757, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.101.61@KRBTEST.COM for kudu/127.19.101.62@KRBTEST.COM
20:09:18.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:09:18.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.19.101.60 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/krb5kdc/kudu_127.19.101.60.keytab.
20:09:18.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.19.101.60 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/krb5kdc/kudu_127.19.101.60.keytab.
20:09:18.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:18.115764 19860 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskQoB_wm/build/tsan/bin/kudu
20:09:18.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskQoB_wm/build/tsan/bin/kudu
20:09:18.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/master-2/wal
20:09:18.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/master-2/data
20:09:18.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:09:18.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:09:18.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:09:18.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:09:18.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:09:18.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:09:18.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/master-2/logs
20:09:18.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/master-2/data/info.pb
20:09:18.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:09:18.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:09:18.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:09:18.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:09:18.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:09:18.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:09:18.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:09:18.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:09:18.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:09:18.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:09:18.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:09:18.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:09:18.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.101.60:35091
20:09:18.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.101.60
20:09:18.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:09:18.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/krb5kdc/kudu_127.19.101.60.keytab
20:09:18.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.19.101.60
20:09:18.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:09:18.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:09:18.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:09:18.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.101.20:37005
20:09:18.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:09:18.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:09:18.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:09:18.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:09:18.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.19.101.62:46347,127.19.101.61:33871,127.19.101.60:35091
20:09:18.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:09:18.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39755/jwks.json
20:09:18.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:09:18.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:18.119412 20034 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.19.101.61" port: 33871 } has no permanent_uuid. Determining permanent_uuid...
20:09:18.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 20:09:18 dist-test-slave-6w59 krb5kdc[19872](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790539757, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.101.61@KRBTEST.COM for kudu/127.19.101.61@KRBTEST.COM
20:09:18.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:18.149456 20034 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.19.101.60" port: 35091 } has no permanent_uuid. Determining permanent_uuid...
20:09:18.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:18.151474 19985 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.19.101.60:35091: connect: Connection refused (error 111)
20:09:18.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:18.155305 20034 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.19.101.60:35091: Network error: Client connection negotiation failed: client connection to 127.19.101.60:35091: connect: Connection refused (error 111)
20:09:18.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:18.204847 20034 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.101.60" port: 35091 } attempt: 1
20:09:18.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:18.208596 20034 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.19.101.60:35091: Network error: Client connection negotiation failed: client connection to 127.19.101.60:35091: connect: Connection refused (error 111)
20:09:18.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:18.277069 20034 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.101.60" port: 35091 } attempt: 2
20:09:18.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:18.280683 20034 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.19.101.60:35091: Network error: Client connection negotiation failed: client connection to 127.19.101.60:35091: connect: Connection refused (error 111)
20:09:18.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:18.372118 20034 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.101.60" port: 35091 } attempt: 3
20:09:18.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:18.375717 20034 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.19.101.60:35091: Network error: Client connection negotiation failed: client connection to 127.19.101.60:35091: connect: Connection refused (error 111)
20:09:18.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:18.426981 20040 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:09:18.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:18.427574 20040 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:09:18.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:18.428277 20040 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:09:18.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:18.461891 20040 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
20:09:18.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:18.462114 20040 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
20:09:18.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:18.462375 20040 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:09:18.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:18.462565 20040 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
20:09:18.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:18.462749 20040 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
20:09:18.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:18.462989 20040 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
20:09:18.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:18.463186 20040 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:39755/jwks.json
20:09:18.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:18.502497 20040 master_runner.cc:386] Master server non-default flags:
20:09:18.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:09:18.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.101.20:37005
20:09:18.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:09:18.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:09:18.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/master-2/data
20:09:18.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/master-2/wal
20:09:18.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
20:09:18.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:09:18.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.19.101.62:46347,127.19.101.61:33871,127.19.101.60:35091
20:09:18.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/krb5kdc/kudu_127.19.101.60.keytab
20:09:18.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.19.101.60
20:09:18.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:09:18.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:09:18.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:09:18.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.101.60:35091
20:09:18.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:09:18.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:09:18.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:09:18.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39755/jwks.json
20:09:18.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:09:18.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:09:18.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:09:18.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:09:18.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:09:18.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/master-2/data/info.pb
20:09:18.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.101.60
20:09:18.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:09:18.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:09:18.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:09:18.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:09:18.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:09:18.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:09:18.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/master-2/logs
20:09:18.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:09:18.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:09:18.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:09:18.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:09:18.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
20:09:18.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
20:09:18.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:09:18.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 19:19:31 UTC on ccycloud.quasar-tlrbor.root.comops.site
20:09:18.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676273
20:09:18.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:09:18.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:18.503824 20040 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:09:18.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:18.504755 20040 file_cache.cc:493] Constructed file cache file cache with capacity 419430
20:09:18.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:18.519064 20034 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.101.60" port: 35091 } attempt: 4
20:09:18.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:18.522537 20050 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:09:18.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:18.523458 20034 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.19.101.60:35091: Network error: Client connection negotiation failed: client connection to 127.19.101.60:35091: connect: Connection refused (error 111)
20:09:18.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:18.538220 19956 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.101.61" port: 33871 } attempt: 7
20:09:18.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 20:09:18 dist-test-slave-6w59 krb5kdc[19872](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790539756, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.101.62@KRBTEST.COM for kudu/127.19.101.61@KRBTEST.COM
20:09:18.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:18.569343 19956 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.19.101.60" port: 35091 } has no permanent_uuid. Determining permanent_uuid...
20:09:18.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:18.575174 19956 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.19.101.60:35091: Network error: Client connection negotiation failed: client connection to 127.19.101.60:35091: connect: Connection refused (error 111)
20:09:18.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:18.633780 19956 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.101.60" port: 35091 } attempt: 1
20:09:18.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:18.639029 19956 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.19.101.60:35091: Network error: Client connection negotiation failed: client connection to 127.19.101.60:35091: connect: Connection refused (error 111)
20:09:18.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:18.720638 19956 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.101.60" port: 35091 } attempt: 2
20:09:18.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:18.725225 19956 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.19.101.60:35091: Network error: Client connection negotiation failed: client connection to 127.19.101.60:35091: connect: Connection refused (error 111)
20:09:18.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:18.810647 19956 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.101.60" port: 35091 } attempt: 3
20:09:18.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:18.815097 19956 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.19.101.60:35091: Network error: Client connection negotiation failed: client connection to 127.19.101.60:35091: connect: Connection refused (error 111)
20:09:18.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:18.822919 20034 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.101.60" port: 35091 } attempt: 5
20:09:18.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:18.826426 20034 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.19.101.60:35091: Network error: Client connection negotiation failed: client connection to 127.19.101.60:35091: connect: Connection refused (error 111)
20:09:18.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:18.955511 19956 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.101.60" port: 35091 } attempt: 4
20:09:18.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:18.959833 19956 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.19.101.60:35091: Network error: Client connection negotiation failed: client connection to 127.19.101.60:35091: connect: Connection refused (error 111)
20:09:19.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:19.243237 19956 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.101.60" port: 35091 } attempt: 5
20:09:19.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:19.247581 19956 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.19.101.60:35091: Network error: Client connection negotiation failed: client connection to 127.19.101.60:35091: connect: Connection refused (error 111)
20:09:19.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:19.374007 20034 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.101.60" port: 35091 } attempt: 6
20:09:19.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:19.381407 20034 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.19.101.60:35091: Network error: Client connection negotiation failed: client connection to 127.19.101.60:35091: connect: Connection refused (error 111)
20:09:19.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:18.523923 20051 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:09:19.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:18.524798 20053 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:09:19.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:18.526335 20040 server_base.cc:1034] running on GCE node
20:09:19.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:19.753737 20040 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:09:19.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:19.757611 20040 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:09:19.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:19.759200 20040 hybrid_clock.cc:648] HybridClock initialized: now 1790539759759168 us; error 54 us; skew 500 ppm
20:09:19.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:19.760272 20040 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:09:19.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 20:09:19 dist-test-slave-6w59 krb5kdc[19872](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790539759, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.101.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:09:19.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:19.767059 20040 init.cc:377] Logged in from keytab as kudu/127.19.101.60@KRBTEST.COM (short username kudu)
20:09:19.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:19.771951 20040 webserver.cc:460] Webserver started at http://127.19.101.60:36285/ using document root <none> and password file <none>
20:09:19.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:19.772925 20040 fs_manager.cc:359] Metadata directory not provided
20:09:19.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:19.773151 20040 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:09:19.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:19.773656 20040 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
20:09:19.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:19.778297 20040 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/master-2/data/instance:
20:09:19.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b0148706b5f84388a19bf06b652c57c9"
20:09:19.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 20:09:19 on dist-test-slave-6w59"
20:09:19.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:19.779439 20040 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/master-2/wal/instance:
20:09:19.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b0148706b5f84388a19bf06b652c57c9"
20:09:19.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 20:09:19 on dist-test-slave-6w59"
20:09:19.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:19.786553 20040 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.006s sys 0.000s
20:09:19.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:19.792258 20066 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:09:19.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:19.793351 20040 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
20:09:19.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:19.793779 20040 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/master-2/data,/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/master-2/wal
20:09:19.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b0148706b5f84388a19bf06b652c57c9"
20:09:19.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 20:09:19 on dist-test-slave-6w59"
20:09:19.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:19.794133 20040 fs_report.cc:383] FS layout report
20:09:19.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:09:19.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/master-2/wal
20:09:19.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/master-2/wal
20:09:19.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/master-2/data/data
20:09:19.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:09:19.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:09:19.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:09:19.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:09:19.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:09:19.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:09:19.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:09:19.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:09:19.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:09:19.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:09:19.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:09:19.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:19.799964 19956 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.101.60" port: 35091 } attempt: 6
20:09:19.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:19.806547 19956 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.19.101.60:35091: Network error: Client connection negotiation failed: client connection to 127.19.101.60:35091: connect: Connection refused (error 111)
20:09:19.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:19.844225 20040 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
20:09:19.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:19.849931 20040 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:09:19.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:19.850407 20040 kserver.cc:163] Server-wide thread pool size limit: 3276
20:09:19.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:19.929193 20040 rpc_server.cc:307] RPC server started. Bound to: 127.19.101.60:35091
20:09:19.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:19.929261 20119 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.19.101.60:35091 every 8 connection(s)
20:09:19.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:19.931912 20040 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/master-2/data/info.pb
20:09:19.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:19.937858 20120 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
20:09:19.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:19.940099 19860 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskQoB_wm/build/tsan/bin/kudu as pid 20040
20:09:19.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:19.940591 19860 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/master-2/wal/instance
20:09:19.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:19.953547 20120 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.19.101.62" port: 46347 } has no permanent_uuid. Determining permanent_uuid...
20:09:19.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 20:09:19 dist-test-slave-6w59 krb5kdc[19872](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790539754, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.19.101.60@KRBTEST.COM
20:09:19.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 20:09:19 dist-test-slave-6w59 krb5kdc[19872](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790539759, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.101.60@KRBTEST.COM for kudu/127.19.101.62@KRBTEST.COM
20:09:19.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:19.992120 20120 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.19.101.61" port: 33871 } has no permanent_uuid. Determining permanent_uuid...
20:09:20.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 20:09:20 dist-test-slave-6w59 krb5kdc[19872](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790539754, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.19.101.61@KRBTEST.COM
20:09:20.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 20:09:20 dist-test-slave-6w59 krb5kdc[19872](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790539759, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.101.60@KRBTEST.COM for kudu/127.19.101.61@KRBTEST.COM
20:09:20.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:20.019711 20120 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.19.101.60" port: 35091 } has no permanent_uuid. Determining permanent_uuid...
20:09:20.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 20:09:20 dist-test-slave-6w59 krb5kdc[19872](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790539759, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.101.60@KRBTEST.COM for kudu/127.19.101.60@KRBTEST.COM
20:09:20.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 20:09:20 dist-test-slave-6w59 krb5kdc[19872](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790539754, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.19.101.62@KRBTEST.COM
20:09:20.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:20.054672 20120 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b0148706b5f84388a19bf06b652c57c9: Bootstrap starting.
20:09:20.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:20.057804 19860 external_mini_cluster.cc:930] 0 TS(s) registered with all masters
20:09:20.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:20.062143 20120 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P b0148706b5f84388a19bf06b652c57c9: Neither blocks nor log segments found. Creating new log.
20:09:20.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:20.065852 20120 log.cc:824] T 00000000000000000000000000000000 P b0148706b5f84388a19bf06b652c57c9: Log is configured to *not* fsync() on all Append() calls
20:09:20.069 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:09:20.070 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
20:09:20.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:20.075947 20120 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b0148706b5f84388a19bf06b652c57c9: No bootstrap required, opened a new log
20:09:20.090 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
20:09:20.091 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
20:09:20.095 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.19.101.62"
port: 46347
}
bound_http_address {
host: "127.19.101.62"
port: 43047
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.19.101.61"
port: 33871
}
bound_http_address {
host: "127.19.101.61"
port: 36079
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.19.101.60"
port: 35091
}
bound_http_address {
host: "127.19.101.60"
port: 36285
}
}
}
20:09:20.095 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
20:09:20.096 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:09:20.096 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
20:09:20.098 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
20:09:20.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:20.105934 20120 raft_consensus.cc:348] T 00000000000000000000000000000000 P b0148706b5f84388a19bf06b652c57c9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d2412d2667ad45169185672e68bf0e7c" member_type: VOTER last_known_addr { host: "127.19.101.62" port: 46347 } } peers { permanent_uuid: "21c7a3e6c1c54335822620013c20745f" member_type: VOTER last_known_addr { host: "127.19.101.61" port: 33871 } } peers { permanent_uuid: "b0148706b5f84388a19bf06b652c57c9" member_type: VOTER last_known_addr { host: "127.19.101.60" port: 35091 } }
20:09:20.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:20.106921 20120 raft_consensus.cc:374] T 00000000000000000000000000000000 P b0148706b5f84388a19bf06b652c57c9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:09:20.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:20.107242 20120 raft_consensus.cc:729] T 00000000000000000000000000000000 P b0148706b5f84388a19bf06b652c57c9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: b0148706b5f84388a19bf06b652c57c9, State: Initialized, Role: FOLLOWER
20:09:20.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:20.108181 20120 consensus_queue.cc:260] T 00000000000000000000000000000000 P b0148706b5f84388a19bf06b652c57c9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d2412d2667ad45169185672e68bf0e7c" member_type: VOTER last_known_addr { host: "127.19.101.62" port: 46347 } } peers { permanent_uuid: "21c7a3e6c1c54335822620013c20745f" member_type: VOTER last_known_addr { host: "127.19.101.61" port: 33871 } } peers { permanent_uuid: "b0148706b5f84388a19bf06b652c57c9" member_type: VOTER last_known_addr { host: "127.19.101.60" port: 35091 } }
20:09:20.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:20.111785 20130 sys_catalog.cc:455] T 00000000000000000000000000000000 P b0148706b5f84388a19bf06b652c57c9 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d2412d2667ad45169185672e68bf0e7c" member_type: VOTER last_known_addr { host: "127.19.101.62" port: 46347 } } peers { permanent_uuid: "21c7a3e6c1c54335822620013c20745f" member_type: VOTER last_known_addr { host: "127.19.101.61" port: 33871 } } peers { permanent_uuid: "b0148706b5f84388a19bf06b652c57c9" member_type: VOTER last_known_addr { host: "127.19.101.60" port: 35091 } } }
20:09:20.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:20.112533 20130 sys_catalog.cc:458] T 00000000000000000000000000000000 P b0148706b5f84388a19bf06b652c57c9 [sys.catalog]: This master's current role is: FOLLOWER
20:09:20.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:20.113816 20120 sys_catalog.cc:564] T 00000000000000000000000000000000 P b0148706b5f84388a19bf06b652c57c9 [sys.catalog]: configured and running, proceeding with master startup.
20:09:20.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 20:09:20 dist-test-slave-6w59 krb5kdc[19872](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790539760, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:09:20.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
20:09:20.141 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:09:20.143 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/krb5kdc/krb5cc
20:09:20.151 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
20:09:20.152 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/krb5kdc/krb5cc
20:09:20.153 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
20:09:20.157 [DEBUG - Time-limited test] (SecurityContext.java:161) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
20:09:20.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:20.162257 20142 catalog_manager.cc:1558] T 00000000000000000000000000000000 P b0148706b5f84388a19bf06b652c57c9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:09:20.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:20.162834 20142 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:09:20.164 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
20:09:20.165 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
20:09:20.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:20.145612 20040 master_runner.cc:186] Error getting master registration for 127.19.101.62:46347: OK, instance_id { permanent_uuid: "d2412d2667ad45169185672e68bf0e7c" instance_seqno: 1790539756156439 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
20:09:20.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:20.178669 20040 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:09:20.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 20:09:20 dist-test-slave-6w59 krb5kdc[19872](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790539760, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:09:20.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
20:09:20.204 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:09:20.205 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/krb5kdc/krb5cc
20:09:20.207 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
20:09:20.207 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.19.101.60:35091
20:09:20.210 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
20:09:20.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:20.210747 19860 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskQoB_wm/build/tsan/bin/kudu with pid 20040
20:09:20.231 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:09:20.231 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.19.101.61:33871
20:09:20.232 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
20:09:20.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:20.232569 19860 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskQoB_wm/build/tsan/bin/kudu with pid 19962
20:09:20.248 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:09:20.248 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.19.101.62:46347
20:09:20.248 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
20:09:20.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:20.249221 19860 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskQoB_wm/build/tsan/bin/kudu with pid 19885
20:09:20.271 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:09:20.271 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.19.101.60:35091
20:09:20.273 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
20:09:20.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:20.275240 19860 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskQoB_wm/build/tsan/bin/kudu
20:09:20.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskQoB_wm/build/tsan/bin/kudu
20:09:20.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/master-2/wal
20:09:20.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/master-2/data
20:09:20.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:09:20.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:09:20.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:09:20.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:09:20.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:09:20.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:09:20.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/master-2/logs
20:09:20.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/master-2/data/info.pb
20:09:20.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:09:20.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:09:20.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:09:20.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:09:20.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:09:20.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:09:20.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:09:20.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:09:20.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:09:20.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:09:20.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:09:20.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:09:20.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.101.60:35091
20:09:20.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.101.60
20:09:20.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36285
20:09:20.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/krb5kdc/kudu_127.19.101.60.keytab
20:09:20.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.19.101.60
20:09:20.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:09:20.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:09:20.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:09:20.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.101.20:37005
20:09:20.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:09:20.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:09:20.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:09:20.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:09:20.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.19.101.62:46347,127.19.101.61:33871,127.19.101.60:35091
20:09:20.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:09:20.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39755/jwks.json
20:09:20.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:09:20.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:20.622010 20146 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:09:20.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:20.622704 20146 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:09:20.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:20.623466 20146 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:09:20.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:20.655355 20146 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
20:09:20.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:20.655644 20146 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
20:09:20.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:20.656054 20146 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:09:20.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:20.656386 20146 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
20:09:20.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:20.656719 20146 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
20:09:20.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:20.657038 20146 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
20:09:20.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:20.657263 20146 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:39755/jwks.json
20:09:20.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:20.694589 20146 master_runner.cc:386] Master server non-default flags:
20:09:20.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:09:20.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.101.20:37005
20:09:20.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:09:20.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:09:20.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/master-2/data
20:09:20.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/master-2/wal
20:09:20.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
20:09:20.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:09:20.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.19.101.62:46347,127.19.101.61:33871,127.19.101.60:35091
20:09:20.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/krb5kdc/kudu_127.19.101.60.keytab
20:09:20.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.19.101.60
20:09:20.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:09:20.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:09:20.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:09:20.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.101.60:35091
20:09:20.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:09:20.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:09:20.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:09:20.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39755/jwks.json
20:09:20.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:09:20.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:09:20.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:09:20.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:09:20.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:09:20.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/master-2/data/info.pb
20:09:20.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.101.60
20:09:20.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36285
20:09:20.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:09:20.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:09:20.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:09:20.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:09:20.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:09:20.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/master-2/logs
20:09:20.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:09:20.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:09:20.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:09:20.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:09:20.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
20:09:20.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
20:09:20.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:09:20.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 19:19:31 UTC on ccycloud.quasar-tlrbor.root.comops.site
20:09:20.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676273
20:09:20.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:09:20.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:20.696180 20146 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:09:20.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:20.697484 20146 file_cache.cc:493] Constructed file cache file cache with capacity 419430
20:09:20.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:20.718401 20154 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:09:21.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:20.718434 20155 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:09:21.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:20.718604 20157 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:09:21.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:20.720762 20146 server_base.cc:1034] running on GCE node
20:09:21.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:21.995740 20146 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:09:21.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:21.998735 20146 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:09:22.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:22.000180 20146 hybrid_clock.cc:648] HybridClock initialized: now 1790539762000146 us; error 41 us; skew 500 ppm
20:09:22.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:22.001089 20146 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:09:22.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 20:09:22 dist-test-slave-6w59 krb5kdc[19872](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790539762, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.101.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:09:22.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:22.006974 20146 init.cc:377] Logged in from keytab as kudu/127.19.101.60@KRBTEST.COM (short username kudu)
20:09:22.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:22.012362 20146 webserver.cc:460] Webserver started at http://127.19.101.60:36285/ using document root <none> and password file <none>
20:09:22.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:22.013325 20146 fs_manager.cc:359] Metadata directory not provided
20:09:22.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:22.013563 20146 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:09:22.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:22.021692 20146 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.006s sys 0.000s
20:09:22.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:22.025946 20165 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:09:22.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:22.027112 20146 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
20:09:22.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:22.027558 20146 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/master-2/data,/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/master-2/wal
20:09:22.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b0148706b5f84388a19bf06b652c57c9"
20:09:22.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 20:09:19 on dist-test-slave-6w59"
20:09:22.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:22.030117 20146 fs_report.cc:383] FS layout report
20:09:22.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:09:22.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/master-2/wal
20:09:22.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/master-2/wal
20:09:22.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/master-2/data/data
20:09:22.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:09:22.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:09:22.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:09:22.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:09:22.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:09:22.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:09:22.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:09:22.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:09:22.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:09:22.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:09:22.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:09:22.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:22.069346 20146 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
20:09:22.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:22.075042 20146 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:09:22.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:22.075555 20146 kserver.cc:163] Server-wide thread pool size limit: 3276
20:09:22.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:22.153333 20146 rpc_server.cc:307] RPC server started. Bound to: 127.19.101.60:35091
20:09:22.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:22.153447 20217 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.19.101.60:35091 every 8 connection(s)
20:09:22.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:22.156173 20146 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/master-2/data/info.pb
20:09:22.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:22.162727 19860 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskQoB_wm/build/tsan/bin/kudu as pid 20146
20:09:22.163 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:09:22.163 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.19.101.61:33871
20:09:22.163 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
20:09:22.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:22.164840 19860 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskQoB_wm/build/tsan/bin/kudu
20:09:22.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskQoB_wm/build/tsan/bin/kudu
20:09:22.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/master-1/wal
20:09:22.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/master-1/data
20:09:22.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:09:22.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:09:22.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:09:22.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:09:22.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:09:22.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:09:22.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/master-1/logs
20:09:22.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/master-1/data/info.pb
20:09:22.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:09:22.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:09:22.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:09:22.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:09:22.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:09:22.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:09:22.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:09:22.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:09:22.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:09:22.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:09:22.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:09:22.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:09:22.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.101.61:33871
20:09:22.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.101.61
20:09:22.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36079
20:09:22.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/krb5kdc/kudu_127.19.101.61.keytab
20:09:22.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.19.101.61
20:09:22.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:09:22.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:09:22.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:09:22.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.101.20:37005
20:09:22.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:09:22.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:09:22.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:09:22.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:09:22.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.19.101.62:46347,127.19.101.61:33871,127.19.101.60:35091
20:09:22.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:09:22.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39755/jwks.json
20:09:22.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:09:22.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:22.166703 20218 sys_catalog.cc:263] Verifying existing consensus state
20:09:22.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:22.173767 20218 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b0148706b5f84388a19bf06b652c57c9: Bootstrap starting.
20:09:22.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:22.205888 20218 log.cc:824] T 00000000000000000000000000000000 P b0148706b5f84388a19bf06b652c57c9: Log is configured to *not* fsync() on all Append() calls
20:09:22.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:22.210511 20218 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b0148706b5f84388a19bf06b652c57c9: Bootstrap replayed 1/1 log segments. Stats: ops{read=0 overwritten=0 applied=0 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
20:09:22.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:22.211292 20218 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b0148706b5f84388a19bf06b652c57c9: Bootstrap complete.
20:09:22.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:22.234680 20218 raft_consensus.cc:348] T 00000000000000000000000000000000 P b0148706b5f84388a19bf06b652c57c9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d2412d2667ad45169185672e68bf0e7c" member_type: VOTER last_known_addr { host: "127.19.101.62" port: 46347 } } peers { permanent_uuid: "21c7a3e6c1c54335822620013c20745f" member_type: VOTER last_known_addr { host: "127.19.101.61" port: 33871 } } peers { permanent_uuid: "b0148706b5f84388a19bf06b652c57c9" member_type: VOTER last_known_addr { host: "127.19.101.60" port: 35091 } }
20:09:22.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:22.237061 20218 raft_consensus.cc:374] T 00000000000000000000000000000000 P b0148706b5f84388a19bf06b652c57c9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:09:22.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:22.237389 20218 raft_consensus.cc:729] T 00000000000000000000000000000000 P b0148706b5f84388a19bf06b652c57c9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: b0148706b5f84388a19bf06b652c57c9, State: Initialized, Role: FOLLOWER
20:09:22.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:22.238443 20218 consensus_queue.cc:260] T 00000000000000000000000000000000 P b0148706b5f84388a19bf06b652c57c9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d2412d2667ad45169185672e68bf0e7c" member_type: VOTER last_known_addr { host: "127.19.101.62" port: 46347 } } peers { permanent_uuid: "21c7a3e6c1c54335822620013c20745f" member_type: VOTER last_known_addr { host: "127.19.101.61" port: 33871 } } peers { permanent_uuid: "b0148706b5f84388a19bf06b652c57c9" member_type: VOTER last_known_addr { host: "127.19.101.60" port: 35091 } }
20:09:22.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:22.241493 20222 sys_catalog.cc:455] T 00000000000000000000000000000000 P b0148706b5f84388a19bf06b652c57c9 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d2412d2667ad45169185672e68bf0e7c" member_type: VOTER last_known_addr { host: "127.19.101.62" port: 46347 } } peers { permanent_uuid: "21c7a3e6c1c54335822620013c20745f" member_type: VOTER last_known_addr { host: "127.19.101.61" port: 33871 } } peers { permanent_uuid: "b0148706b5f84388a19bf06b652c57c9" member_type: VOTER last_known_addr { host: "127.19.101.60" port: 35091 } } }
20:09:22.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:22.242357 20222 sys_catalog.cc:458] T 00000000000000000000000000000000 P b0148706b5f84388a19bf06b652c57c9 [sys.catalog]: This master's current role is: FOLLOWER
20:09:22.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:22.243189 20218 sys_catalog.cc:564] T 00000000000000000000000000000000 P b0148706b5f84388a19bf06b652c57c9 [sys.catalog]: configured and running, proceeding with master startup.
20:09:22.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:22.283561 20233 catalog_manager.cc:1558] T 00000000000000000000000000000000 P b0148706b5f84388a19bf06b652c57c9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:09:22.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:22.283923 20233 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:09:22.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:22.281572 20146 master_runner.cc:186] Error getting master registration for 127.19.101.62:46347: Network error: Client connection negotiation failed: client connection to 127.19.101.62:46347: connect: Connection refused (error 111),
20:09:22.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:22.312048 20146 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:09:22.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:22.330709 20171 tablet.cc:2369] T 00000000000000000000000000000000 P b0148706b5f84388a19bf06b652c57c9: Can't schedule compaction. Clean time has not been advanced past its initial value.
20:09:22.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:22.420497 20222 raft_consensus.cc:482] T 00000000000000000000000000000000 P b0148706b5f84388a19bf06b652c57c9 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:09:22.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:22.421860 20222 raft_consensus.cc:504] T 00000000000000000000000000000000 P b0148706b5f84388a19bf06b652c57c9 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d2412d2667ad45169185672e68bf0e7c" member_type: VOTER last_known_addr { host: "127.19.101.62" port: 46347 } } peers { permanent_uuid: "21c7a3e6c1c54335822620013c20745f" member_type: VOTER last_known_addr { host: "127.19.101.61" port: 33871 } } peers { permanent_uuid: "b0148706b5f84388a19bf06b652c57c9" member_type: VOTER last_known_addr { host: "127.19.101.60" port: 35091 } }
20:09:22.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:22.427593 20222 leader_election.cc:288] T 00000000000000000000000000000000 P b0148706b5f84388a19bf06b652c57c9 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers d2412d2667ad45169185672e68bf0e7c (127.19.101.62:46347), 21c7a3e6c1c54335822620013c20745f (127.19.101.61:33871)
20:09:22.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:22.428110 20168 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.19.101.62:46347: connect: Connection refused (error 111)
20:09:22.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:22.443101 20169 leader_election.cc:334] T 00000000000000000000000000000000 P b0148706b5f84388a19bf06b652c57c9 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 21c7a3e6c1c54335822620013c20745f (127.19.101.61:33871): Network error: Client connection negotiation failed: client connection to 127.19.101.61:33871: connect: Connection refused (error 111)
20:09:22.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:22.445638 20168 leader_election.cc:334] T 00000000000000000000000000000000 P b0148706b5f84388a19bf06b652c57c9 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer d2412d2667ad45169185672e68bf0e7c (127.19.101.62:46347): Network error: Client connection negotiation failed: client connection to 127.19.101.62:46347: connect: Connection refused (error 111)
20:09:22.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:22.446090 20168 leader_election.cc:302] T 00000000000000000000000000000000 P b0148706b5f84388a19bf06b652c57c9 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: b0148706b5f84388a19bf06b652c57c9; no voters: 21c7a3e6c1c54335822620013c20745f, d2412d2667ad45169185672e68bf0e7c
20:09:22.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:22.447206 20222 raft_consensus.cc:2726] T 00000000000000000000000000000000 P b0148706b5f84388a19bf06b652c57c9 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
20:09:22.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:22.545395 20220 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:09:22.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:22.546507 20220 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:09:22.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:22.547546 20220 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:09:22.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:22.582172 20220 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
20:09:22.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:22.583045 20220 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
20:09:22.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:22.583518 20220 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:09:22.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:22.583736 20220 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
20:09:22.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:22.583964 20220 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
20:09:22.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:22.584151 20220 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
20:09:22.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:22.584290 20220 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:39755/jwks.json
20:09:22.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:22.622593 20220 master_runner.cc:386] Master server non-default flags:
20:09:22.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:09:22.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.101.20:37005
20:09:22.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:09:22.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:09:22.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/master-1/data
20:09:22.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/master-1/wal
20:09:22.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
20:09:22.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:09:22.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.19.101.62:46347,127.19.101.61:33871,127.19.101.60:35091
20:09:22.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/krb5kdc/kudu_127.19.101.61.keytab
20:09:22.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.19.101.61
20:09:22.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:09:22.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:09:22.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:09:22.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.101.61:33871
20:09:22.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:09:22.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:09:22.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:09:22.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39755/jwks.json
20:09:22.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:09:22.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:09:22.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:09:22.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:09:22.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:09:22.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/master-1/data/info.pb
20:09:22.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.101.61
20:09:22.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36079
20:09:22.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:09:22.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:09:22.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:09:22.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:09:22.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:09:22.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/master-1/logs
20:09:22.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:09:22.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:09:22.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:09:22.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:09:22.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
20:09:22.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
20:09:22.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:09:22.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 19:19:31 UTC on ccycloud.quasar-tlrbor.root.comops.site
20:09:22.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676273
20:09:22.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:09:22.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:22.624099 20220 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:09:22.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:22.625445 20220 file_cache.cc:493] Constructed file cache file cache with capacity 419430
20:09:22.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:22.646257 20246 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:09:23.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:23.288054 20233 catalog_manager.cc:1558] T 00000000000000000000000000000000 P b0148706b5f84388a19bf06b652c57c9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:09:23.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:23.288648 20233 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:09:24.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:24.230999 20251 raft_consensus.cc:482] T 00000000000000000000000000000000 P b0148706b5f84388a19bf06b652c57c9 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:09:24.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:24.232206 20251 raft_consensus.cc:504] T 00000000000000000000000000000000 P b0148706b5f84388a19bf06b652c57c9 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d2412d2667ad45169185672e68bf0e7c" member_type: VOTER last_known_addr { host: "127.19.101.62" port: 46347 } } peers { permanent_uuid: "21c7a3e6c1c54335822620013c20745f" member_type: VOTER last_known_addr { host: "127.19.101.61" port: 33871 } } peers { permanent_uuid: "b0148706b5f84388a19bf06b652c57c9" member_type: VOTER last_known_addr { host: "127.19.101.60" port: 35091 } }
20:09:24.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:24.249603 20251 leader_election.cc:288] T 00000000000000000000000000000000 P b0148706b5f84388a19bf06b652c57c9 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers d2412d2667ad45169185672e68bf0e7c (127.19.101.62:46347), 21c7a3e6c1c54335822620013c20745f (127.19.101.61:33871)
20:09:24.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:24.261349 20168 leader_election.cc:334] T 00000000000000000000000000000000 P b0148706b5f84388a19bf06b652c57c9 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer d2412d2667ad45169185672e68bf0e7c (127.19.101.62:46347): Network error: Client connection negotiation failed: client connection to 127.19.101.62:46347: connect: Connection refused (error 111)
20:09:24.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:24.264526 20169 leader_election.cc:334] T 00000000000000000000000000000000 P b0148706b5f84388a19bf06b652c57c9 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 21c7a3e6c1c54335822620013c20745f (127.19.101.61:33871): Network error: Client connection negotiation failed: client connection to 127.19.101.61:33871: connect: Connection refused (error 111)
20:09:24.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:24.265164 20169 leader_election.cc:302] T 00000000000000000000000000000000 P b0148706b5f84388a19bf06b652c57c9 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: b0148706b5f84388a19bf06b652c57c9; no voters: 21c7a3e6c1c54335822620013c20745f, d2412d2667ad45169185672e68bf0e7c
20:09:24.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:24.266515 20251 raft_consensus.cc:2726] T 00000000000000000000000000000000 P b0148706b5f84388a19bf06b652c57c9 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
20:09:24.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:24.293198 20233 catalog_manager.cc:1558] T 00000000000000000000000000000000 P b0148706b5f84388a19bf06b652c57c9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:09:24.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:24.293807 20233 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:09:24.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:22.646301 20244 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:09:24.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:22.649427 20243 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:09:24.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:24.391234 20245 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1743 milliseconds
20:09:24.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:24.391343 20220 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:09:24.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:24.392683 20220 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:09:24.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:24.395690 20220 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:09:24.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:24.397099 20220 hybrid_clock.cc:648] HybridClock initialized: now 1790539764397001 us; error 113 us; skew 500 ppm
20:09:24.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:24.398020 20220 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:09:24.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 20:09:24 dist-test-slave-6w59 krb5kdc[19872](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790539764, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.101.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:09:24.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:24.403833 20220 init.cc:377] Logged in from keytab as kudu/127.19.101.61@KRBTEST.COM (short username kudu)
20:09:24.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:24.410079 20220 webserver.cc:460] Webserver started at http://127.19.101.61:36079/ using document root <none> and password file <none>
20:09:24.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:24.410985 20220 fs_manager.cc:359] Metadata directory not provided
20:09:24.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:24.411255 20220 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:09:24.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:24.419464 20220 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.007s sys 0.001s
20:09:24.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:24.424105 20260 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:09:24.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:24.425223 20220 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.004s sys 0.001s
20:09:24.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:24.425498 20220 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/master-1/data,/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/master-1/wal
20:09:24.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "21c7a3e6c1c54335822620013c20745f"
20:09:24.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 20:09:17 on dist-test-slave-6w59"
20:09:24.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:24.428144 20220 fs_report.cc:383] FS layout report
20:09:24.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:09:24.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/master-1/wal
20:09:24.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/master-1/wal
20:09:24.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/master-1/data/data
20:09:24.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:09:24.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:09:24.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:09:24.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:09:24.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:09:24.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:09:24.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:09:24.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:09:24.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:09:24.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:09:24.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:09:24.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:24.476542 20220 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
20:09:24.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:24.482930 20220 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:09:24.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:24.483373 20220 kserver.cc:163] Server-wide thread pool size limit: 3276
20:09:24.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:24.577229 20220 rpc_server.cc:307] RPC server started. Bound to: 127.19.101.61:33871
20:09:24.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:24.577314 20311 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.19.101.61:33871 every 8 connection(s)
20:09:24.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:24.580780 20220 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/master-1/data/info.pb
20:09:24.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:24.589985 19860 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskQoB_wm/build/tsan/bin/kudu as pid 20220
20:09:24.591 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:09:24.591 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.19.101.62:46347
20:09:24.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:24.591657 20312 sys_catalog.cc:263] Verifying existing consensus state
20:09:24.591 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
20:09:24.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260927 20:09:24.592557 20312 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
20:09:24.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:24.593014 20220 master.cc:537] Master@127.19.101.61:33871 shutting down...
20:09:24.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:24.593714 19860 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskQoB_wm/build/tsan/bin/kudu
20:09:24.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskQoB_wm/build/tsan/bin/kudu
20:09:24.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/master-0/wal
20:09:24.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/master-0/data
20:09:24.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:09:24.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:09:24.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:09:24.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:09:24.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:09:24.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:09:24.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/master-0/logs
20:09:24.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/master-0/data/info.pb
20:09:24.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:09:24.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:09:24.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:09:24.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:09:24.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:09:24.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:09:24.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:09:24.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:09:24.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:09:24.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:09:24.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:09:24.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:09:24.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.101.62:46347
20:09:24.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.101.62
20:09:24.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=43047
20:09:24.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/krb5kdc/kudu_127.19.101.62.keytab
20:09:24.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.19.101.62
20:09:24.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:09:24.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:09:24.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:09:24.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.101.20:37005
20:09:24.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:09:24.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:09:24.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:09:24.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:09:24.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.19.101.62:46347,127.19.101.61:33871,127.19.101.60:35091
20:09:24.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:09:24.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39755/jwks.json
20:09:24.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:09:24.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:24.643215 20220 master.cc:552] Master@127.19.101.61:33871 shutdown complete.
20:09:24.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
20:09:24.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:24.912688 20314 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:09:24.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:24.913360 20314 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:09:24.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:24.914106 20314 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:09:24.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:24.946484 20314 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
20:09:24.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:24.946767 20314 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
20:09:24.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:24.947223 20314 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:09:24.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:24.947551 20314 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
20:09:24.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:24.947870 20314 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
20:09:24.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:24.948182 20314 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
20:09:24.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:24.948410 20314 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:39755/jwks.json
20:09:24.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:24.987017 20314 master_runner.cc:386] Master server non-default flags:
20:09:24.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:09:24.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.101.20:37005
20:09:24.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:09:24.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:09:24.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/master-0/data
20:09:24.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/master-0/wal
20:09:24.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
20:09:24.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:09:24.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.19.101.62:46347,127.19.101.61:33871,127.19.101.60:35091
20:09:24.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/krb5kdc/kudu_127.19.101.62.keytab
20:09:24.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.19.101.62
20:09:24.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:09:24.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:09:24.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:09:24.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.101.62:46347
20:09:24.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:09:24.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:09:24.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:09:24.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39755/jwks.json
20:09:24.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:09:24.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:09:24.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:09:24.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:09:24.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:09:24.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/master-0/data/info.pb
20:09:24.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.101.62
20:09:24.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=43047
20:09:24.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:09:24.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:09:24.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:09:24.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:09:24.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:09:24.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/master-0/logs
20:09:24.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:09:24.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:09:24.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:09:24.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:09:24.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
20:09:24.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
20:09:24.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:09:24.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 19:19:31 UTC on ccycloud.quasar-tlrbor.root.comops.site
20:09:24.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676273
20:09:24.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:09:24.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:24.988521 20314 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:09:24.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:24.989877 20314 file_cache.cc:493] Constructed file cache file cache with capacity 419430
20:09:25.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:25.007144 20322 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:09:25.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:25.297008 20233 catalog_manager.cc:1558] T 00000000000000000000000000000000 P b0148706b5f84388a19bf06b652c57c9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:09:25.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:25.297274 20233 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:09:26.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:25.007246 20321 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:09:26.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:25.007645 20324 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:09:26.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:26.241443 20323 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
20:09:26.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:26.241554 20314 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:09:26.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:26.242682 20314 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:09:26.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:26.245937 20314 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:09:26.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:26.247339 20314 hybrid_clock.cc:648] HybridClock initialized: now 1790539766247296 us; error 60 us; skew 500 ppm
20:09:26.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:26.248101 20314 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:09:26.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 20:09:26 dist-test-slave-6w59 krb5kdc[19872](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790539766, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.101.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:09:26.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:26.254160 20314 init.cc:377] Logged in from keytab as kudu/127.19.101.62@KRBTEST.COM (short username kudu)
20:09:26.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:26.259472 20314 webserver.cc:460] Webserver started at http://127.19.101.62:43047/ using document root <none> and password file <none>
20:09:26.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:26.260314 20314 fs_manager.cc:359] Metadata directory not provided
20:09:26.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:26.260501 20314 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:09:26.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:26.268227 20314 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.000s sys 0.004s
20:09:26.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:26.272235 20332 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:09:26.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:26.273208 20314 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.001s
20:09:26.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:26.273569 20314 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/master-0/data,/tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/master-0/wal
20:09:26.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d2412d2667ad45169185672e68bf0e7c"
20:09:26.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 20:09:16 on dist-test-slave-6w59"
20:09:26.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:26.276032 20314 fs_report.cc:383] FS layout report
20:09:26.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:09:26.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/master-0/wal
20:09:26.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/master-0/wal
20:09:26.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/master-0/data/data
20:09:26.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:09:26.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:09:26.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:09:26.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:09:26.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:09:26.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:09:26.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:09:26.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:09:26.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:09:26.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:09:26.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:09:26.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:26.301178 20233 catalog_manager.cc:1558] T 00000000000000000000000000000000 P b0148706b5f84388a19bf06b652c57c9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:09:26.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:26.301534 20233 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:09:26.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:26.327057 20314 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
20:09:26.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:26.332587 20314 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:09:26.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:26.333091 20314 kserver.cc:163] Server-wide thread pool size limit: 3276
20:09:26.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:26.408517 20314 rpc_server.cc:307] RPC server started. Bound to: 127.19.101.62:46347
20:09:26.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:26.408666 20383 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.19.101.62:46347 every 8 connection(s)
20:09:26.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:26.411407 20314 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/master-0/data/info.pb
20:09:26.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:26.417241 19860 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskQoB_wm/build/tsan/bin/kudu as pid 20314
20:09:26.418 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:09:26.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:26.423017 20384 sys_catalog.cc:263] Verifying existing consensus state
20:09:26.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260927 20:09:26.423964 20384 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
20:09:26.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:26.424422 20314 master.cc:537] Master@127.19.101.62:46347 shutting down...
20:09:26.483 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.19.101.60:35091(127.19.101.60:35091)] connecting to peer
20:09:26.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:26.515610 20314 master.cc:552] Master@127.19.101.62:46347 shutdown complete.
20:09:26.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskQoB_wm/test-tmp/mini-kudu-cluster7402296206606094035/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
20:09:26.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:26.535612 20387 raft_consensus.cc:482] T 00000000000000000000000000000000 P b0148706b5f84388a19bf06b652c57c9 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:09:26.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:26.536087 20387 raft_consensus.cc:504] T 00000000000000000000000000000000 P b0148706b5f84388a19bf06b652c57c9 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d2412d2667ad45169185672e68bf0e7c" member_type: VOTER last_known_addr { host: "127.19.101.62" port: 46347 } } peers { permanent_uuid: "21c7a3e6c1c54335822620013c20745f" member_type: VOTER last_known_addr { host: "127.19.101.61" port: 33871 } } peers { permanent_uuid: "b0148706b5f84388a19bf06b652c57c9" member_type: VOTER last_known_addr { host: "127.19.101.60" port: 35091 } }
20:09:26.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:26.545001 20387 leader_election.cc:288] T 00000000000000000000000000000000 P b0148706b5f84388a19bf06b652c57c9 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers d2412d2667ad45169185672e68bf0e7c (127.19.101.62:46347), 21c7a3e6c1c54335822620013c20745f (127.19.101.61:33871)
20:09:26.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:26.553665 20169 leader_election.cc:334] T 00000000000000000000000000000000 P b0148706b5f84388a19bf06b652c57c9 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 21c7a3e6c1c54335822620013c20745f (127.19.101.61:33871): Network error: Client connection negotiation failed: client connection to 127.19.101.61:33871: connect: Connection refused (error 111)
20:09:26.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:26.555866 20168 leader_election.cc:334] T 00000000000000000000000000000000 P b0148706b5f84388a19bf06b652c57c9 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer d2412d2667ad45169185672e68bf0e7c (127.19.101.62:46347): Network error: Client connection negotiation failed: client connection to 127.19.101.62:46347: connect: Connection refused (error 111)
20:09:26.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:26.556252 20168 leader_election.cc:302] T 00000000000000000000000000000000 P b0148706b5f84388a19bf06b652c57c9 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: b0148706b5f84388a19bf06b652c57c9; no voters: 21c7a3e6c1c54335822620013c20745f, d2412d2667ad45169185672e68bf0e7c
20:09:26.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:26.557528 20387 raft_consensus.cc:2726] T 00000000000000000000000000000000 P b0148706b5f84388a19bf06b652c57c9 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
20:09:26.560 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.19.101.61:33871(127.19.101.61:33871)] connecting to peer
20:09:26.562 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.19.101.62:46347(127.19.101.62:46347)] connecting to peer
20:09:26.608 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871
20:09:26.608 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.101.61:33871(127.19.101.61:33871)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871
20:09:26.610 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.101.61:33871: Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871
20:09:26.610 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.19.101.60:35091(127.19.101.60:35091)] Successfully connected to peer
20:09:26.663 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347
20:09:26.663 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.101.62:46347(127.19.101.62:46347)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347
20:09:26.664 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.101.62:46347: Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347
20:09:26.679 [WARN - kudu-nio-0] (SecurityContext.java:209) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
20:09:26.687 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.19.101.60
20:09:26.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 20:09:26 dist-test-slave-6w59 krb5kdc[19872](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790539760, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.19.101.60@KRBTEST.COM
20:09:26.911 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xd6cc2103, L:/127.0.0.1:36166 - R:/127.19.101.60:35091] using SASL/GSSAPI
20:09:26.927 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.101.60:35091,127.19.101.61:33871,127.19.101.62:46347; will retry
20:09:26.947 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.19.101.61:33871(127.19.101.61:33871)] connecting to peer
20:09:26.950 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.19.101.62:46347(127.19.101.62:46347)] connecting to peer
20:09:26.950 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871
20:09:26.950 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.101.61:33871(127.19.101.61:33871)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871
20:09:26.951 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.101.61:33871: Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871
20:09:26.955 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347
20:09:26.955 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.101.62:46347(127.19.101.62:46347)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347
20:09:26.956 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.101.62:46347: Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347
20:09:26.956 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.101.60:35091,127.19.101.61:33871,127.19.101.62:46347; will retry
20:09:26.963 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.19.101.61:33871(127.19.101.61:33871)] connecting to peer
20:09:26.964 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.19.101.62:46347(127.19.101.62:46347)] connecting to peer
20:09:26.966 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871
20:09:26.966 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.101.61:33871(127.19.101.61:33871)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871
20:09:26.967 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.101.61:33871: Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871
20:09:26.968 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347
20:09:26.968 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.101.62:46347(127.19.101.62:46347)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347
20:09:26.968 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.101.62:46347: Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347
20:09:26.969 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.101.60:35091,127.19.101.61:33871,127.19.101.62:46347; will retry
20:09:26.984 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.19.101.61:33871(127.19.101.61:33871)] connecting to peer
20:09:26.986 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.19.101.62:46347(127.19.101.62:46347)] connecting to peer
20:09:26.988 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871
20:09:26.988 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.101.61:33871(127.19.101.61:33871)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871
20:09:26.988 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.101.61:33871: Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871
20:09:26.990 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347
20:09:26.990 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.101.62:46347(127.19.101.62:46347)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347
20:09:26.991 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.101.62:46347: Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347
20:09:26.992 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.101.60:35091,127.19.101.61:33871,127.19.101.62:46347; will retry
20:09:27.004 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.19.101.61:33871(127.19.101.61:33871)] connecting to peer
20:09:27.005 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.19.101.62:46347(127.19.101.62:46347)] connecting to peer
20:09:27.007 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871
20:09:27.007 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.101.61:33871(127.19.101.61:33871)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871
20:09:27.008 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.101.61:33871: Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871
20:09:27.010 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347
20:09:27.010 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.101.62:46347(127.19.101.62:46347)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347
20:09:27.011 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.101.62:46347: Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347
20:09:27.011 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.101.60:35091,127.19.101.61:33871,127.19.101.62:46347; will retry
20:09:27.024 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.19.101.61:33871(127.19.101.61:33871)] connecting to peer
20:09:27.026 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.19.101.62:46347(127.19.101.62:46347)] connecting to peer
20:09:27.030 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871
20:09:27.030 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.101.61:33871(127.19.101.61:33871)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871
20:09:27.031 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.101.61:33871: Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871
20:09:27.037 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347
20:09:27.037 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.101.62:46347(127.19.101.62:46347)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347
20:09:27.038 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.101.62:46347: Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347
20:09:27.038 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.101.60:35091,127.19.101.61:33871,127.19.101.62:46347; will retry
20:09:27.104 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.19.101.61:33871(127.19.101.61:33871)] connecting to peer
20:09:27.106 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.19.101.62:46347(127.19.101.62:46347)] connecting to peer
20:09:27.107 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871
20:09:27.107 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.101.61:33871(127.19.101.61:33871)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871
20:09:27.108 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.101.61:33871: Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871
20:09:27.110 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347
20:09:27.110 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.101.62:46347(127.19.101.62:46347)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347
20:09:27.111 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.101.62:46347: Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347
20:09:27.111 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.101.60:35091,127.19.101.61:33871,127.19.101.62:46347; will retry
20:09:27.184 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.19.101.61:33871(127.19.101.61:33871)] connecting to peer
20:09:27.187 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871
20:09:27.187 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.19.101.62:46347(127.19.101.62:46347)] connecting to peer
20:09:27.187 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.101.61:33871(127.19.101.61:33871)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871
20:09:27.188 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.101.61:33871: Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871
20:09:27.190 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347
20:09:27.190 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.101.62:46347(127.19.101.62:46347)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347
20:09:27.190 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.101.62:46347: Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347
20:09:27.191 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.101.60:35091,127.19.101.61:33871,127.19.101.62:46347; will retry
20:09:27.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:27.304368 20233 catalog_manager.cc:1558] T 00000000000000000000000000000000 P b0148706b5f84388a19bf06b652c57c9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:09:27.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:27.304625 20233 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:09:27.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:27.314507 20146 master_runner.cc:186] Error getting master registration for 127.19.101.62:46347: Network error: Client connection negotiation failed: client connection to 127.19.101.62:46347: connect: Connection refused (error 111),
20:09:27.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:27.314828 20146 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:09:27.444 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.19.101.61:33871(127.19.101.61:33871)] connecting to peer
20:09:27.445 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.19.101.62:46347(127.19.101.62:46347)] connecting to peer
20:09:27.446 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871
20:09:27.447 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.101.61:33871(127.19.101.61:33871)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871
20:09:27.447 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.101.61:33871: Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871
20:09:27.449 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347
20:09:27.449 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.101.62:46347(127.19.101.62:46347)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347
20:09:27.450 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.101.62:46347: Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347
20:09:27.450 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.101.60:35091,127.19.101.61:33871,127.19.101.62:46347; will retry
20:09:27.924 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.19.101.61:33871(127.19.101.61:33871)] connecting to peer
20:09:27.927 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.19.101.62:46347(127.19.101.62:46347)] connecting to peer
20:09:27.927 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871
20:09:27.927 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.101.61:33871(127.19.101.61:33871)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871
20:09:27.928 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.101.61:33871: Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871
20:09:27.929 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347
20:09:27.930 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.101.62:46347(127.19.101.62:46347)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347
20:09:27.930 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.101.62:46347: Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347
20:09:27.931 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.101.60:35091,127.19.101.61:33871,127.19.101.62:46347; will retry
20:09:28.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:28.307690 20233 catalog_manager.cc:1558] T 00000000000000000000000000000000 P b0148706b5f84388a19bf06b652c57c9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:09:28.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:28.307955 20233 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:09:28.644 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.19.101.61:33871(127.19.101.61:33871)] connecting to peer
20:09:28.646 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.19.101.62:46347(127.19.101.62:46347)] connecting to peer
20:09:28.647 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347
20:09:28.647 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.101.62:46347(127.19.101.62:46347)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347
20:09:28.647 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871
20:09:28.647 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.101.61:33871(127.19.101.61:33871)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871
20:09:28.647 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.101.62:46347: Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347
20:09:28.648 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.101.61:33871: Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871
20:09:28.650 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.101.60:35091,127.19.101.61:33871,127.19.101.62:46347; will retry
20:09:29.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:29.310631 20233 catalog_manager.cc:1558] T 00000000000000000000000000000000 P b0148706b5f84388a19bf06b652c57c9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:09:29.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:29.310962 20233 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:09:29.964 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.19.101.61:33871(127.19.101.61:33871)] connecting to peer
20:09:29.966 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.19.101.62:46347(127.19.101.62:46347)] connecting to peer
20:09:29.968 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871
20:09:29.968 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.101.61:33871(127.19.101.61:33871)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871
20:09:29.968 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.101.61:33871: Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871
20:09:29.969 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347
20:09:29.969 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.101.62:46347(127.19.101.62:46347)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347
20:09:29.969 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.101.62:46347: Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347
20:09:29.970 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.101.60:35091,127.19.101.61:33871,127.19.101.62:46347; will retry
20:09:29.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:29.983776 20400 raft_consensus.cc:482] T 00000000000000000000000000000000 P b0148706b5f84388a19bf06b652c57c9 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:09:29.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:29.984176 20400 raft_consensus.cc:504] T 00000000000000000000000000000000 P b0148706b5f84388a19bf06b652c57c9 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d2412d2667ad45169185672e68bf0e7c" member_type: VOTER last_known_addr { host: "127.19.101.62" port: 46347 } } peers { permanent_uuid: "21c7a3e6c1c54335822620013c20745f" member_type: VOTER last_known_addr { host: "127.19.101.61" port: 33871 } } peers { permanent_uuid: "b0148706b5f84388a19bf06b652c57c9" member_type: VOTER last_known_addr { host: "127.19.101.60" port: 35091 } }
20:09:29.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:29.985661 20400 leader_election.cc:288] T 00000000000000000000000000000000 P b0148706b5f84388a19bf06b652c57c9 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers d2412d2667ad45169185672e68bf0e7c (127.19.101.62:46347), 21c7a3e6c1c54335822620013c20745f (127.19.101.61:33871)
20:09:29.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:29.987469 20168 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.19.101.62:46347: connect: Connection refused (error 111) [suppressed 5 similar messages]
20:09:29.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:29.990253 20168 leader_election.cc:334] T 00000000000000000000000000000000 P b0148706b5f84388a19bf06b652c57c9 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer d2412d2667ad45169185672e68bf0e7c (127.19.101.62:46347): Network error: Client connection negotiation failed: client connection to 127.19.101.62:46347: connect: Connection refused (error 111)
20:09:29.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:29.990619 20169 leader_election.cc:334] T 00000000000000000000000000000000 P b0148706b5f84388a19bf06b652c57c9 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 21c7a3e6c1c54335822620013c20745f (127.19.101.61:33871): Network error: Client connection negotiation failed: client connection to 127.19.101.61:33871: connect: Connection refused (error 111)
20:09:29.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:29.991016 20169 leader_election.cc:302] T 00000000000000000000000000000000 P b0148706b5f84388a19bf06b652c57c9 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: b0148706b5f84388a19bf06b652c57c9; no voters: 21c7a3e6c1c54335822620013c20745f, d2412d2667ad45169185672e68bf0e7c
20:09:29.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:29.991640 20400 raft_consensus.cc:2726] T 00000000000000000000000000000000 P b0148706b5f84388a19bf06b652c57c9 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
20:09:30.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:30.313809 20233 catalog_manager.cc:1558] T 00000000000000000000000000000000 P b0148706b5f84388a19bf06b652c57c9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:09:30.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:30.314062 20233 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:09:31.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:31.317463 20233 catalog_manager.cc:1558] T 00000000000000000000000000000000 P b0148706b5f84388a19bf06b652c57c9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:09:31.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:31.317780 20233 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:09:31.984 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.19.101.61:33871(127.19.101.61:33871)] connecting to peer
20:09:31.986 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.19.101.62:46347(127.19.101.62:46347)] connecting to peer
20:09:31.986 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871
20:09:31.986 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.101.61:33871(127.19.101.61:33871)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871
20:09:31.988 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.101.61:33871: Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871
20:09:31.988 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347
20:09:31.989 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.101.62:46347(127.19.101.62:46347)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347
20:09:31.989 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.101.62:46347: Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347
20:09:31.989 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.101.60:35091,127.19.101.61:33871,127.19.101.62:46347; will retry
20:09:32.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 20:09:32 dist-test-slave-6w59 krb5kdc[19872](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790539772, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.101.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:09:32.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:32.010838 20161 init.cc:295] Successfully reacquired a new kerberos TGT
20:09:32.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:32.317641 20146 master_runner.cc:186] Error getting master registration for 127.19.101.62:46347: Network error: Client connection negotiation failed: client connection to 127.19.101.62:46347: connect: Connection refused (error 111),
20:09:32.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:32.318050 20146 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:09:32.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:32.320551 20233 catalog_manager.cc:1558] T 00000000000000000000000000000000 P b0148706b5f84388a19bf06b652c57c9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:09:32.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:32.320755 20233 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:09:33.004 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.19.101.61:33871(127.19.101.61:33871)] connecting to peer
20:09:33.005 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.19.101.62:46347(127.19.101.62:46347)] connecting to peer
20:09:33.006 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871
20:09:33.007 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.101.61:33871(127.19.101.61:33871)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871
20:09:33.007 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.101.61:33871: Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871
20:09:33.008 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347
20:09:33.008 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.101.62:46347(127.19.101.62:46347)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347
20:09:33.008 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.101.62:46347: Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347
20:09:33.008 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.101.60:35091,127.19.101.61:33871,127.19.101.62:46347; will retry
20:09:33.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:33.323535 20233 catalog_manager.cc:1558] T 00000000000000000000000000000000 P b0148706b5f84388a19bf06b652c57c9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:09:33.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:33.323789 20233 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:09:34.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:34.101679 20411 raft_consensus.cc:482] T 00000000000000000000000000000000 P b0148706b5f84388a19bf06b652c57c9 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:09:34.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:34.102288 20411 raft_consensus.cc:504] T 00000000000000000000000000000000 P b0148706b5f84388a19bf06b652c57c9 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d2412d2667ad45169185672e68bf0e7c" member_type: VOTER last_known_addr { host: "127.19.101.62" port: 46347 } } peers { permanent_uuid: "21c7a3e6c1c54335822620013c20745f" member_type: VOTER last_known_addr { host: "127.19.101.61" port: 33871 } } peers { permanent_uuid: "b0148706b5f84388a19bf06b652c57c9" member_type: VOTER last_known_addr { host: "127.19.101.60" port: 35091 } }
20:09:34.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:34.104210 20411 leader_election.cc:288] T 00000000000000000000000000000000 P b0148706b5f84388a19bf06b652c57c9 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers d2412d2667ad45169185672e68bf0e7c (127.19.101.62:46347), 21c7a3e6c1c54335822620013c20745f (127.19.101.61:33871)
20:09:34.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:34.109543 20168 leader_election.cc:334] T 00000000000000000000000000000000 P b0148706b5f84388a19bf06b652c57c9 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer d2412d2667ad45169185672e68bf0e7c (127.19.101.62:46347): Network error: Client connection negotiation failed: client connection to 127.19.101.62:46347: connect: Connection refused (error 111)
20:09:34.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:34.110131 20169 leader_election.cc:334] T 00000000000000000000000000000000 P b0148706b5f84388a19bf06b652c57c9 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 21c7a3e6c1c54335822620013c20745f (127.19.101.61:33871): Network error: Client connection negotiation failed: client connection to 127.19.101.61:33871: connect: Connection refused (error 111)
20:09:34.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:34.110424 20169 leader_election.cc:302] T 00000000000000000000000000000000 P b0148706b5f84388a19bf06b652c57c9 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: b0148706b5f84388a19bf06b652c57c9; no voters: 21c7a3e6c1c54335822620013c20745f, d2412d2667ad45169185672e68bf0e7c
20:09:34.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:34.111187 20411 raft_consensus.cc:2726] T 00000000000000000000000000000000 P b0148706b5f84388a19bf06b652c57c9 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
20:09:34.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:34.326539 20233 catalog_manager.cc:1558] T 00000000000000000000000000000000 P b0148706b5f84388a19bf06b652c57c9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:09:34.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:34.326766 20233 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:09:35.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:35.329509 20233 catalog_manager.cc:1558] T 00000000000000000000000000000000 P b0148706b5f84388a19bf06b652c57c9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:09:35.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:35.329752 20233 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:09:35.804 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.19.101.61:33871(127.19.101.61:33871)] connecting to peer
20:09:35.805 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.19.101.62:46347(127.19.101.62:46347)] connecting to peer
20:09:35.807 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871
20:09:35.807 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.101.61:33871(127.19.101.61:33871)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871
20:09:35.807 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.101.61:33871: Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871
20:09:35.809 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347
20:09:35.809 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.101.62:46347(127.19.101.62:46347)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347
20:09:35.809 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.101.62:46347: Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347
20:09:35.809 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.101.60:35091,127.19.101.61:33871,127.19.101.62:46347; will retry
20:09:36.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:36.332669 20233 catalog_manager.cc:1558] T 00000000000000000000000000000000 P b0148706b5f84388a19bf06b652c57c9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:09:36.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:36.332979 20233 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:09:37.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:37.120558 20416 raft_consensus.cc:482] T 00000000000000000000000000000000 P b0148706b5f84388a19bf06b652c57c9 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:09:37.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:37.121023 20416 raft_consensus.cc:504] T 00000000000000000000000000000000 P b0148706b5f84388a19bf06b652c57c9 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d2412d2667ad45169185672e68bf0e7c" member_type: VOTER last_known_addr { host: "127.19.101.62" port: 46347 } } peers { permanent_uuid: "21c7a3e6c1c54335822620013c20745f" member_type: VOTER last_known_addr { host: "127.19.101.61" port: 33871 } } peers { permanent_uuid: "b0148706b5f84388a19bf06b652c57c9" member_type: VOTER last_known_addr { host: "127.19.101.60" port: 35091 } }
20:09:37.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:37.122581 20416 leader_election.cc:288] T 00000000000000000000000000000000 P b0148706b5f84388a19bf06b652c57c9 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers d2412d2667ad45169185672e68bf0e7c (127.19.101.62:46347), 21c7a3e6c1c54335822620013c20745f (127.19.101.61:33871)
20:09:37.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:37.124837 20169 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.19.101.61:33871: connect: Connection refused (error 111) [suppressed 3 similar messages]
20:09:37.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:37.128083 20169 leader_election.cc:334] T 00000000000000000000000000000000 P b0148706b5f84388a19bf06b652c57c9 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 21c7a3e6c1c54335822620013c20745f (127.19.101.61:33871): Network error: Client connection negotiation failed: client connection to 127.19.101.61:33871: connect: Connection refused (error 111)
20:09:37.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:37.128545 20168 leader_election.cc:334] T 00000000000000000000000000000000 P b0148706b5f84388a19bf06b652c57c9 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer d2412d2667ad45169185672e68bf0e7c (127.19.101.62:46347): Network error: Client connection negotiation failed: client connection to 127.19.101.62:46347: connect: Connection refused (error 111)
20:09:37.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:37.128930 20168 leader_election.cc:302] T 00000000000000000000000000000000 P b0148706b5f84388a19bf06b652c57c9 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: b0148706b5f84388a19bf06b652c57c9; no voters: 21c7a3e6c1c54335822620013c20745f, d2412d2667ad45169185672e68bf0e7c
20:09:37.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:37.129586 20416 raft_consensus.cc:2726] T 00000000000000000000000000000000 P b0148706b5f84388a19bf06b652c57c9 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
20:09:37.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:37.320475 20146 master_runner.cc:186] Error getting master registration for 127.19.101.62:46347: Network error: Client connection negotiation failed: client connection to 127.19.101.62:46347: connect: Connection refused (error 111),
20:09:37.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:37.320915 20146 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:09:37.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:37.335785 20233 catalog_manager.cc:1558] T 00000000000000000000000000000000 P b0148706b5f84388a19bf06b652c57c9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:09:37.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:37.336053 20233 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:09:38.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:38.338922 20233 catalog_manager.cc:1558] T 00000000000000000000000000000000 P b0148706b5f84388a19bf06b652c57c9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:09:38.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:38.339273 20233 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:09:38.384 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.19.101.61:33871(127.19.101.61:33871)] connecting to peer
20:09:38.385 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.19.101.62:46347(127.19.101.62:46347)] connecting to peer
20:09:38.387 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871
20:09:38.387 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.101.61:33871(127.19.101.61:33871)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871
20:09:38.387 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.101.61:33871: Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871
20:09:38.389 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347
20:09:38.389 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.101.62:46347(127.19.101.62:46347)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347
20:09:38.389 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.101.62:46347: Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347
20:09:38.389 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.101.60:35091,127.19.101.61:33871,127.19.101.62:46347; will retry
20:09:39.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:39.342101 20233 catalog_manager.cc:1558] T 00000000000000000000000000000000 P b0148706b5f84388a19bf06b652c57c9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:09:39.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:39.342409 20233 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:09:40.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:40.345613 20233 catalog_manager.cc:1558] T 00000000000000000000000000000000 P b0148706b5f84388a19bf06b652c57c9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:09:40.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:40.345917 20233 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:09:41.064 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.19.101.61:33871(127.19.101.61:33871)] connecting to peer
20:09:41.066 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.19.101.62:46347(127.19.101.62:46347)] connecting to peer
20:09:41.066 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871
20:09:41.066 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.101.61:33871(127.19.101.61:33871)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871
20:09:41.067 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.101.61:33871: Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871
20:09:41.069 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347
20:09:41.069 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.101.62:46347(127.19.101.62:46347)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347
20:09:41.069 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.101.62:46347: Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347
20:09:41.069 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.101.60:35091,127.19.101.61:33871,127.19.101.62:46347; will retry
20:09:41.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:41.349263 20233 catalog_manager.cc:1558] T 00000000000000000000000000000000 P b0148706b5f84388a19bf06b652c57c9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:09:41.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:41.349654 20233 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:09:42.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 20:09:42 dist-test-slave-6w59 krb5kdc[19872](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790539782, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.101.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:09:42.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:42.013669 20161 init.cc:295] Successfully reacquired a new kerberos TGT
20:09:42.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:42.324172 20146 master_runner.cc:186] Error getting master registration for 127.19.101.62:46347: Network error: Client connection negotiation failed: client connection to 127.19.101.62:46347: connect: Connection refused (error 111),
20:09:42.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:42.324630 20146 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:09:42.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:42.352447 20233 catalog_manager.cc:1558] T 00000000000000000000000000000000 P b0148706b5f84388a19bf06b652c57c9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:09:42.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:42.352757 20233 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:09:43.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:43.355581 20233 catalog_manager.cc:1558] T 00000000000000000000000000000000 P b0148706b5f84388a19bf06b652c57c9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:09:43.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:43.355858 20233 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:09:43.964 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.19.101.61:33871(127.19.101.61:33871)] connecting to peer
20:09:43.966 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.19.101.62:46347(127.19.101.62:46347)] connecting to peer
20:09:43.967 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871
20:09:43.967 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.101.61:33871(127.19.101.61:33871)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871
20:09:43.967 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.101.61:33871: Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871
20:09:43.969 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347
20:09:43.969 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.101.62:46347(127.19.101.62:46347)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347
20:09:43.969 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.101.62:46347: Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347
20:09:43.970 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.101.60:35091,127.19.101.61:33871,127.19.101.62:46347; will retry
20:09:44.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:44.358548 20233 catalog_manager.cc:1558] T 00000000000000000000000000000000 P b0148706b5f84388a19bf06b652c57c9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:09:44.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:44.358845 20233 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:09:45.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:45.361545 20233 catalog_manager.cc:1558] T 00000000000000000000000000000000 P b0148706b5f84388a19bf06b652c57c9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:09:45.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:45.361826 20233 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:09:46.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:46.365397 20233 catalog_manager.cc:1558] T 00000000000000000000000000000000 P b0148706b5f84388a19bf06b652c57c9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:09:46.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:46.365742 20233 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:09:47.005 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.19.101.61:33871(127.19.101.61:33871)] connecting to peer
20:09:47.007 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.19.101.62:46347(127.19.101.62:46347)] connecting to peer
20:09:47.007 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871
20:09:47.007 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.101.61:33871(127.19.101.61:33871)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871
20:09:47.008 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.101.61:33871: Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871
20:09:47.009 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347
20:09:47.010 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.101.62:46347(127.19.101.62:46347)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347
20:09:47.010 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.101.62:46347: Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347
20:09:47.010 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.101.60:35091,127.19.101.61:33871,127.19.101.62:46347; will retry
20:09:47.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:47.217931 20432 raft_consensus.cc:482] T 00000000000000000000000000000000 P b0148706b5f84388a19bf06b652c57c9 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:09:47.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:47.218432 20432 raft_consensus.cc:504] T 00000000000000000000000000000000 P b0148706b5f84388a19bf06b652c57c9 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d2412d2667ad45169185672e68bf0e7c" member_type: VOTER last_known_addr { host: "127.19.101.62" port: 46347 } } peers { permanent_uuid: "21c7a3e6c1c54335822620013c20745f" member_type: VOTER last_known_addr { host: "127.19.101.61" port: 33871 } } peers { permanent_uuid: "b0148706b5f84388a19bf06b652c57c9" member_type: VOTER last_known_addr { host: "127.19.101.60" port: 35091 } }
20:09:47.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:47.220259 20432 leader_election.cc:288] T 00000000000000000000000000000000 P b0148706b5f84388a19bf06b652c57c9 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers d2412d2667ad45169185672e68bf0e7c (127.19.101.62:46347), 21c7a3e6c1c54335822620013c20745f (127.19.101.61:33871)
20:09:47.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:47.221980 20168 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.19.101.62:46347: connect: Connection refused (error 111) [suppressed 1 similar messages]
20:09:47.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:47.224660 20168 leader_election.cc:334] T 00000000000000000000000000000000 P b0148706b5f84388a19bf06b652c57c9 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer d2412d2667ad45169185672e68bf0e7c (127.19.101.62:46347): Network error: Client connection negotiation failed: client connection to 127.19.101.62:46347: connect: Connection refused (error 111)
20:09:47.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:47.225129 20169 leader_election.cc:334] T 00000000000000000000000000000000 P b0148706b5f84388a19bf06b652c57c9 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 21c7a3e6c1c54335822620013c20745f (127.19.101.61:33871): Network error: Client connection negotiation failed: client connection to 127.19.101.61:33871: connect: Connection refused (error 111)
20:09:47.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:47.225389 20169 leader_election.cc:302] T 00000000000000000000000000000000 P b0148706b5f84388a19bf06b652c57c9 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: b0148706b5f84388a19bf06b652c57c9; no voters: 21c7a3e6c1c54335822620013c20745f, d2412d2667ad45169185672e68bf0e7c
20:09:47.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:47.225963 20432 raft_consensus.cc:2726] T 00000000000000000000000000000000 P b0148706b5f84388a19bf06b652c57c9 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
20:09:47.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:47.326543 20146 master_runner.cc:186] Error getting master registration for 127.19.101.62:46347: Network error: Client connection negotiation failed: client connection to 127.19.101.62:46347: connect: Connection refused (error 111),
20:09:47.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:47.326988 20146 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:09:47.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:47.369621 20233 catalog_manager.cc:1558] T 00000000000000000000000000000000 P b0148706b5f84388a19bf06b652c57c9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:09:47.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:47.369951 20233 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:09:48.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:48.373504 20233 catalog_manager.cc:1558] T 00000000000000000000000000000000 P b0148706b5f84388a19bf06b652c57c9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:09:48.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:48.373736 20233 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:09:49.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:49.376417 20233 catalog_manager.cc:1558] T 00000000000000000000000000000000 P b0148706b5f84388a19bf06b652c57c9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:09:49.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:49.376665 20233 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:09:49.424 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.19.101.61:33871(127.19.101.61:33871)] connecting to peer
20:09:49.425 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.19.101.62:46347(127.19.101.62:46347)] connecting to peer
20:09:49.427 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871
20:09:49.427 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.101.61:33871(127.19.101.61:33871)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871
20:09:49.427 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.101.61:33871: Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871
20:09:49.428 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347
20:09:49.428 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.101.62:46347(127.19.101.62:46347)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347
20:09:49.428 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.101.62:46347: Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347
20:09:49.428 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.101.60:35091,127.19.101.61:33871,127.19.101.62:46347; will retry
20:09:50.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:50.379420 20233 catalog_manager.cc:1558] T 00000000000000000000000000000000 P b0148706b5f84388a19bf06b652c57c9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:09:50.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:50.379659 20233 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:09:51.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:51.384127 20233 catalog_manager.cc:1558] T 00000000000000000000000000000000 P b0148706b5f84388a19bf06b652c57c9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:09:51.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:51.384516 20233 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:09:52.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 20:09:52 dist-test-slave-6w59 krb5kdc[19872](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790539792, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.101.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:09:52.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:52.016129 20161 init.cc:295] Successfully reacquired a new kerberos TGT
20:09:52.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:52.330041 20146 master_runner.cc:186] Error getting master registration for 127.19.101.62:46347: Network error: Client connection negotiation failed: client connection to 127.19.101.62:46347: connect: Connection refused (error 111),
20:09:52.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:52.330436 20146 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:09:52.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:52.387204 20233 catalog_manager.cc:1558] T 00000000000000000000000000000000 P b0148706b5f84388a19bf06b652c57c9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:09:52.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:52.387454 20233 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:09:52.484 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.19.101.61:33871(127.19.101.61:33871)] connecting to peer
20:09:52.485 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.19.101.62:46347(127.19.101.62:46347)] connecting to peer
20:09:52.486 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871
20:09:52.486 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347
20:09:52.486 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.101.61:33871(127.19.101.61:33871)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871
20:09:52.486 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.101.62:46347(127.19.101.62:46347)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347
20:09:52.486 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.101.61:33871: Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871
20:09:52.486 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.101.62:46347: Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347
20:09:52.488 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.101.60:35091,127.19.101.61:33871,127.19.101.62:46347; will retry
20:09:53.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:53.390094 20233 catalog_manager.cc:1558] T 00000000000000000000000000000000 P b0148706b5f84388a19bf06b652c57c9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:09:53.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:53.390380 20233 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:09:54.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:54.392926 20233 catalog_manager.cc:1558] T 00000000000000000000000000000000 P b0148706b5f84388a19bf06b652c57c9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:09:54.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:54.393167 20233 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:09:55.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:55.395675 20233 catalog_manager.cc:1558] T 00000000000000000000000000000000 P b0148706b5f84388a19bf06b652c57c9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:09:55.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 20:09:55.396013 20233 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:09:55.444 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.19.101.61:33871(127.19.101.61:33871)] connecting to peer
20:09:55.446 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.19.101.62:46347(127.19.101.62:46347)] connecting to peer
20:09:55.446 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871
20:09:55.446 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.101.61:33871(127.19.101.61:33871)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871
20:09:55.446 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.101.61:33871: Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871
20:09:55.448 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347
20:09:55.449 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.101.62:46347(127.19.101.62:46347)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347
20:09:55.449 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.101.62:46347: Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347
20:09:55.449 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.101.60:35091,127.19.101.61:33871,127.19.101.62:46347; will retry
20:09:55.454 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=22, TimeoutTracker(timeout=30000, elapsed=29027), Traces: [0ms] refreshing cache from master, [45ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.101.60:35091, [132ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.101.61:33871, [133ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.101.62:46347, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.101.61:33871: Network error: Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871, [235ms] Sub RPC ConnectToMaster: received response from server master-127.19.101.62:46347: Network error: Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347, [496ms] Sub RPC ConnectToMaster: received response from server master-127.19.101.60:35091: OK, [500ms] delaying RPC due to: Service unavailable: Master config (127.19.101.60:35091,127.19.101.61:33871,127.19.101.62:46347) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347, [516ms] refreshing cache from master, [516ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.101.60:35091, [519ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.101.61:33871, [522ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.101.62:46347, [523ms] Sub RPC ConnectToMaster: received response from server master-127.19.101.61:33871: Network error: Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871, [525ms] Sub RPC ConnectToMaster: received response from server master-127.19.101.60:35091: OK, [527ms] Sub RPC ConnectToMaster: received response from server master-127.19.101.62:46347: Network error: Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347, [529ms] delaying RPC due to: Service unavailable: Master config (127.19.101.60:35091,127.19.101.61:33871,127.19.101.62:46347) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347, [535ms] refreshing cache from master, [535ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.101.60:35091, [535ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.101.61:33871, [536ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.101.62:46347, [538ms] Sub RPC ConnectToMaster: received response from server master-127.19.101.61:33871: Network error: Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871, [539ms] Sub RPC ConnectToMaster: received response from server master-127.19.101.60:35091: OK, [540ms] Sub RPC ConnectToMaster: received response from server master-127.19.101.62:46347: Network error: Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347, [541ms] delaying RPC due to: Service unavailable: Master config (127.19.101.60:35091,127.19.101.61:33871,127.19.101.62:46347) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347, [555ms] refreshing cache from master, [556ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.101.60:35091, [556ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.101.61:33871, [557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.101.62:46347, [560ms] Sub RPC ConnectToMaster: received response from server master-127.19.101.60:35091: OK, [560ms] Sub RPC ConnectToMaster: received response from server master-127.19.101.61:33871: Network error: Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871, [562ms] Sub RPC ConnectToMaster: received response from server master-127.19.101.62:46347: Network error: Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347, [564ms] delaying RPC due to: Service unavailable: Master config (127.19.101.60:35091,127.19.101.61:33871,127.19.101.62:46347) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347, [576ms] refreshing cache from master, [576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.101.60:35091, [576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.101.61:33871, [577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.101.62:46347, [580ms] Sub RPC ConnectToMaster: received response from server master-127.19.101.61:33871: Network error: Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871, [582ms] Sub RPC ConnectToMaster: received response from server master-127.19.101.60:35091: OK, [583ms] Sub RPC ConnectToMaster: received response from server master-127.19.101.62:46347: Network error: Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347, [584ms] delaying RPC due to: Service unavailable: Master config (127.19.101.60:35091,127.19.101.61:33871,127.19.101.62:46347) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347, [596ms] refreshing cache from master, [596ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.101.60:35091, [596ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.101.61:33871, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.101.62:46347, [602ms] Sub RPC ConnectToMaster: received response from server master-127.19.101.61:33871: Network error: Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871, [603ms] Sub RPC ConnectToMaster: received response from server master-127.19.101.60:35091: OK, [609ms] Sub RPC ConnectToMaster: received response from server master-127.19.101.62:46347: Network error: Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347, [611ms] delaying RPC due to: Service unavailable: Master config (127.19.101.60:35091,127.19.101.61:33871,127.19.101.62:46347) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347, [675ms] refreshing cache from master, [675ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.101.60:35091, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.101.61:33871, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.101.62:46347, [679ms] Sub RPC ConnectToMaster: received response from server master-127.19.101.61:33871: Network error: Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871, [681ms] Sub RPC ConnectToMaster: received response from server master-127.19.101.60:35091: OK, [682ms] Sub RPC ConnectToMaster: received response from server master-127.19.101.62:46347: Network error: Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347, [683ms] delaying RPC due to: Service unavailable: Master config (127.19.101.60:35091,127.19.101.61:33871,127.19.101.62:46347) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347, [755ms] refreshing cache from master, [756ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.101.60:35091, [756ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.101.61:33871, [758ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.101.62:46347, [759ms] Sub RPC ConnectToMaster: received response from server master-127.19.101.61:33871: Network error: Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871, [760ms] Sub RPC ConnectToMaster: received response from server master-127.19.101.60:35091: OK, [762ms] Sub RPC ConnectToMaster: received response from server master-127.19.101.62:46347: Network error: Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347, [763ms] delaying RPC due to: Service unavailable: Master config (127.19.101.60:35091,127.19.101.61:33871,127.19.101.62:46347) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347, [1015ms] refreshing cache from master, [1016ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.101.60:35091, [1016ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.101.61:33871, [1017ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.101.62:46347, [1019ms] Sub RPC ConnectToMaster: received response from server master-127.19.101.61:33871: Network error: Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871, [1020ms] Sub RPC ConnectToMaster: received response from server master-127.19.101.60:35091: OK, [1021ms] Sub RPC ConnectToMaster: received response from server master-127.19.101.62:46347: Network error: Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347, [1022ms] delaying RPC due to: Service unavailable: Master config (127.19.101.60:35091,127.19.101.61:33871,127.19.101.62:46347) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347, [1495ms] refreshing cache from master, [1496ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.101.60:35091, [1496ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.101.61:33871, [1498ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.101.62:46347, [1500ms] Sub RPC ConnectToMaster: received response from server master-127.19.101.61:33871: Network error: Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871, [1501ms] Sub RPC ConnectToMaster: received response from server master-127.19.101.60:35091: OK, [1502ms] Sub RPC ConnectToMaster: received response from server master-127.19.101.62:46347: Network error: Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347, [1503ms] delaying RPC due to: Service unavailable: Master config (127.19.101.60:35091,127.19.101.61:33871,127.19.101.62:46347) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347, [2215ms] refreshing cache from master, [2216ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.101.60:35091, [2216ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.101.61:33871, [2218ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.101.62:46347, [2219ms] Sub RPC ConnectToMaster: received response from server master-127.19.101.62:46347: Network error: Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347, [2219ms] Sub RPC ConnectToMaster: received response from server master-127.19.101.61:33871: Network error: Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871, [2222ms] Sub RPC ConnectToMaster: received response from server master-127.19.101.60:35091: OK, [2222ms] delaying RPC due to: Service unavailable: Master config (127.19.101.60:35091,127.19.101.61:33871,127.19.101.62:46347) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871, [3536ms] refreshing cache from master, [3536ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.101.60:35091, [3536ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.101.61:33871, [3538ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.101.62:46347, [3540ms] Sub RPC ConnectToMaster: received response from server master-127.19.101.60:35091: OK, [3540ms] Sub RPC ConnectToMaster: received response from server master-127.19.101.61:33871: Network error: Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871, [3541ms] Sub RPC ConnectToMaster: received response from server master-127.19.101.62:46347: Network error: Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347, [3542ms] delaying RPC due to: Service unavailable: Master config (127.19.101.60:35091,127.19.101.61:33871,127.19.101.62:46347) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347, [5555ms] refreshing cache from master, [5555ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.101.60:35091, [5555ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.101.61:33871, [5557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.101.62:46347, [5559ms] too many traces: truncated at 100 traces, deferred=Deferred@1639445259(state=PENDING, result=null, callback=(continuation of Deferred@1405902449 after retry RPC after error@173232753) -> (continuation of Deferred@213191538 after retry RPC after error@1380224580) -> (continuation of Deferred@1674214827 after retry RPC after error@1931111289) -> (continuation of Deferred@1464865011 after retry RPC after error@563122931) -> (continuation of Deferred@294097999 after retry RPC after error@828798117) -> (continuation of Deferred@1726365170 after retry RPC after error@1128541399) -> (continuation of Deferred@1850535196 after retry RPC after error@1812823852) -> (continuation of Deferred@1003734770 after retry RPC after error@1502836683) -> (continuation of Deferred@1354651112 after retry RPC after error@1473098956) -> (continuation of Deferred@1921784977 after retry RPC after error@811798564) -> (continuation of Deferred@315622101 after retry RPC after error@671203223) -> (continuation of Deferred@981161618 after retry RPC after error@305238985) -> (continuation of Deferred@6684144 after retry RPC after error@624760684) -> (continuation of Deferred@1167202975 after retry RPC after error@1537435199) -> (continuation of Deferred@1954974406 after retry RPC after error@969766221) -> (continuation of Deferred@1818150409 after retry RPC after error@739034915) -> (continuation of Deferred@1822210270 after retry RPC after error@1042931485) -> (continuation of Deferred@618477663 after retry RPC after error@1847772728) -> (continuation of Deferred@1735514474 after retry RPC after error@399520686) -> (continuation of Deferred@1518530251 after retry RPC after error@585756837) -> (continuation of Deferred@1270531867 after retry RPC after error@1566691457), errback=(continuation of Deferred@1405902449 after retry RPC after error@173232753) -> (continuation of Deferred@213191538 after retry RPC after error@1380224580) -> (continuation of Deferred@1674214827 after retry RPC after error@1931111289) -> (continuation of Deferred@1464865011 after retry RPC after error@563122931) -> (continuation of Deferred@294097999 after retry RPC after error@828798117) -> (continuation of Deferred@1726365170 after retry RPC after error@1128541399) -> (continuation of Deferred@1850535196 after retry RPC after error@1812823852) -> (continuation of Deferred@1003734770 after retry RPC after error@1502836683) -> (continuation of Deferred@1354651112 after retry RPC after error@1473098956) -> (continuation of Deferred@1921784977 after retry RPC after error@811798564) -> (continuation of Deferred@315622101 after retry RPC after error@671203223) -> (continuation of Deferred@981161618 after retry RPC after error@305238985) -> (continuation of Deferred@6684144 after retry RPC after error@624760684) -> (continuation of Deferred@1167202975 after retry RPC after error@1537435199) -> (continuation of Deferred@1954974406 after retry RPC after error@969766221) -> (continuation of Deferred@1818150409 after retry RPC after error@739034915) -> (continuation of Deferred@1822210270 after retry RPC after error@1042931485) -> (continuation of Deferred@618477663 after retry RPC after error@1847772728) -> (continuation of Deferred@1735514474 after retry RPC after error@399520686) -> (continuation of Deferred@1518530251 after retry RPC after error@585756837) -> (continuation of Deferred@1270531867 after retry RPC after error@1566691457)))
20:09:55.461 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.19.101.60:35091(127.19.101.60:35091)] disconnecting while in state READY
20:09:55.462 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.19.101.62:46347(127.19.101.62:46347)] disconnecting while in state TERMINATED
20:09:55.462 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.19.101.61:33871(127.19.101.61:33871)] disconnecting while in state TERMINATED
20:09:55.469 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2704) Releasing all remaining resources
20:09:55.475 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.19.101.60:35091(127.19.101.60:35091)] handling channelInactive
20:09:55.476 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.101.60:35091(127.19.101.60:35091)] cleaning up while in state READY due to: connection closed
20:09:55.477 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
20:09:55.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:55.482801 19860 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskQoB_wm/build/tsan/bin/kudu with pid 20146
20:09:56.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-27T20:09:56Z chronyd exiting
20:09:57.707 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
20:09:57.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:57.708494 19134 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskQoB_wm/build/tsan/bin/kudu with pid 19425
20:09:57.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:57.733229 19134 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskQoB_wm/build/tsan/bin/kudu with pid 19571
20:09:57.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:57.760087 19134 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskQoB_wm/build/tsan/bin/kudu with pid 19719
20:09:57.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:57.787640 19134 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskQoB_wm/build/tsan/bin/kudu with pid 19146
20:09:57.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:57.815778 19134 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskQoB_wm/build/tsan/bin/kudu with pid 19217
20:09:57.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 20:09:57.844985 19134 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskQoB_wm/build/tsan/bin/kudu with pid 19293
20:09:58.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-27T20:09:58Z chronyd exiting
20:10:00.020 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=22, TimeoutTracker(timeout=30000, elapsed=29027), Traces: [0ms] refreshing cache from master, [45ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.101.60:35091, [132ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.101.61:33871, [133ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.101.62:46347, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.101.61:33871: Network error: Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871, [235ms] Sub RPC ConnectToMaster: received response from server master-127.19.101.62:46347: Network error: Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347, [496ms] Sub RPC ConnectToMaster: received response from server master-127.19.101.60:35091: OK, [500ms] delaying RPC due to: Service unavailable: Master config (127.19.101.60:35091,127.19.101.61:33871,127.19.101.62:46347) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347, [516ms] refreshing cache from master, [516ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.101.60:35091, [519ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.101.61:33871, [522ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.101.62:46347, [523ms] Sub RPC ConnectToMaster: received response from server master-127.19.101.61:33871: Network error: Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871, [525ms] Sub RPC ConnectToMaster: received response from server master-127.19.101.60:35091: OK, [527ms] Sub RPC ConnectToMaster: received response from server master-127.19.101.62:46347: Network error: Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347, [529ms] delaying RPC due to: Service unavailable: Master config (127.19.101.60:35091,127.19.101.61:33871,127.19.101.62:46347) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347, [535ms] refreshing cache from master, [535ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.101.60:35091, [535ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.101.61:33871, [536ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.101.62:46347, [538ms] Sub RPC ConnectToMaster: received response from server master-127.19.101.61:33871: Network error: Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871, [539ms] Sub RPC ConnectToMaster: received response from server master-127.19.101.60:35091: OK, [540ms] Sub RPC ConnectToMaster: received response from server master-127.19.101.62:46347: Network error: Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347, [541ms] delaying RPC due to: Service unavailable: Master config (127.19.101.60:35091,127.19.101.61:33871,127.19.101.62:46347) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347, [555ms] refreshing cache from master, [556ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.101.60:35091, [556ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.101.61:33871, [557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.101.62:46347, [560ms] Sub RPC ConnectToMaster: received response from server master-127.19.101.60:35091: OK, [560ms] Sub RPC ConnectToMaster: received response from server master-127.19.101.61:33871: Network error: Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871, [562ms] Sub RPC ConnectToMaster: received response from server master-127.19.101.62:46347: Network error: Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347, [564ms] delaying RPC due to: Service unavailable: Master config (127.19.101.60:35091,127.19.101.61:33871,127.19.101.62:46347) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347, [576ms] refreshing cache from master, [576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.101.60:35091, [576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.101.61:33871, [577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.101.62:46347, [580ms] Sub RPC ConnectToMaster: received response from server master-127.19.101.61:33871: Network error: Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871, [582ms] Sub RPC ConnectToMaster: received response from server master-127.19.101.60:35091: OK, [583ms] Sub RPC ConnectToMaster: received response from server master-127.19.101.62:46347: Network error: Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347, [584ms] delaying RPC due to: Service unavailable: Master config (127.19.101.60:35091,127.19.101.61:33871,127.19.101.62:46347) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347, [596ms] refreshing cache from master, [596ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.101.60:35091, [596ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.101.61:33871, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.101.62:46347, [602ms] Sub RPC ConnectToMaster: received response from server master-127.19.101.61:33871: Network error: Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871, [603ms] Sub RPC ConnectToMaster: received response from server master-127.19.101.60:35091: OK, [609ms] Sub RPC ConnectToMaster: received response from server master-127.19.101.62:46347: Network error: Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347, [611ms] delaying RPC due to: Service unavailable: Master config (127.19.101.60:35091,127.19.101.61:33871,127.19.101.62:46347) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347, [675ms] refreshing cache from master, [675ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.101.60:35091, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.101.61:33871, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.101.62:46347, [679ms] Sub RPC ConnectToMaster: received response from server master-127.19.101.61:33871: Network error: Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871, [681ms] Sub RPC ConnectToMaster: received response from server master-127.19.101.60:35091: OK, [682ms] Sub RPC ConnectToMaster: received response from server master-127.19.101.62:46347: Network error: Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347, [683ms] delaying RPC due to: Service unavailable: Master config (127.19.101.60:35091,127.19.101.61:33871,127.19.101.62:46347) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347, [755ms] refreshing cache from master, [756ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.101.60:35091, [756ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.101.61:33871, [758ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.101.62:46347, [759ms] Sub RPC ConnectToMaster: received response from server master-127.19.101.61:33871: Network error: Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871, [760ms] Sub RPC ConnectToMaster: received response from server master-127.19.101.60:35091: OK, [762ms] Sub RPC ConnectToMaster: received response from server master-127.19.101.62:46347: Network error: Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347, [763ms] delaying RPC due to: Service unavailable: Master config (127.19.101.60:35091,127.19.101.61:33871,127.19.101.62:46347) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347, [1015ms] refreshing cache from master, [1016ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.101.60:35091, [1016ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.101.61:33871, [1017ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.101.62:46347, [1019ms] Sub RPC ConnectToMaster: received response from server master-127.19.101.61:33871: Network error: Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871, [1020ms] Sub RPC ConnectToMaster: received response from server master-127.19.101.60:35091: OK, [1021ms] Sub RPC ConnectToMaster: received response from server master-127.19.101.62:46347: Network error: Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347, [1022ms] delaying RPC due to: Service unavailable: Master config (127.19.101.60:35091,127.19.101.61:33871,127.19.101.62:46347) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347, [1495ms] refreshing cache from master, [1496ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.101.60:35091, [1496ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.101.61:33871, [1498ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.101.62:46347, [1500ms] Sub RPC ConnectToMaster: received response from server master-127.19.101.61:33871: Network error: Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871, [1501ms] Sub RPC ConnectToMaster: received response from server master-127.19.101.60:35091: OK, [1502ms] Sub RPC ConnectToMaster: received response from server master-127.19.101.62:46347: Network error: Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347, [1503ms] delaying RPC due to: Service unavailable: Master config (127.19.101.60:35091,127.19.101.61:33871,127.19.101.62:46347) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347, [2215ms] refreshing cache from master, [2216ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.101.60:35091, [2216ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.101.61:33871, [2218ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.101.62:46347, [2219ms] Sub RPC ConnectToMaster: received response from server master-127.19.101.62:46347: Network error: Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347, [2219ms] Sub RPC ConnectToMaster: received response from server master-127.19.101.61:33871: Network error: Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871, [2222ms] Sub RPC ConnectToMaster: received response from server master-127.19.101.60:35091: OK, [2222ms] delaying RPC due to: Service unavailable: Master config (127.19.101.60:35091,127.19.101.61:33871,127.19.101.62:46347) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871, [3536ms] refreshing cache from master, [3536ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.101.60:35091, [3536ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.101.61:33871, [3538ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.101.62:46347, [3540ms] Sub RPC ConnectToMaster: received response from server master-127.19.101.60:35091: OK, [3540ms] Sub RPC ConnectToMaster: received response from server master-127.19.101.61:33871: Network error: Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871, [3541ms] Sub RPC ConnectToMaster: received response from server master-127.19.101.62:46347: Network error: Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347, [3542ms] delaying RPC due to: Service unavailable: Master config (127.19.101.60:35091,127.19.101.61:33871,127.19.101.62:46347) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347, [5555ms] refreshing cache from master, [5555ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.101.60:35091, [5555ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.101.61:33871, [5557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.101.62:46347, [5559ms] too many traces: truncated at 100 traces, deferred=Deferred@1639445259(state=PENDING, result=null, callback=(continuation of Deferred@1405902449 after retry RPC after error@173232753) -> (continuation of Deferred@213191538 after retry RPC after error@1380224580) -> (continuation of Deferred@1674214827 after retry RPC after error@1931111289) -> (continuation of Deferred@1464865011 after retry RPC after error@563122931) -> (continuation of Deferred@294097999 after retry RPC after error@828798117) -> (continuation of Deferred@1726365170 after retry RPC after error@1128541399) -> (continuation of Deferred@1850535196 after retry RPC after error@1812823852) -> (continuation of Deferred@1003734770 after retry RPC after error@1502836683) -> (continuation of Deferred@1354651112 after retry RPC after error@1473098956) -> (continuation of Deferred@1921784977 after retry RPC after error@811798564) -> (continuation of Deferred@315622101 after retry RPC after error@671203223) -> (continuation of Deferred@981161618 after retry RPC after error@305238985) -> (continuation of Deferred@6684144 after retry RPC after error@624760684) -> (continuation of Deferred@1167202975 after retry RPC after error@1537435199) -> (continuation of Deferred@1954974406 after retry RPC after error@969766221) -> (continuation of Deferred@1818150409 after retry RPC after error@739034915) -> (continuation of Deferred@1822210270 after retry RPC after error@1042931485) -> (continuation of Deferred@618477663 after retry RPC after error@1847772728) -> (continuation of Deferred@1735514474 after retry RPC after error@399520686) -> (continuation of Deferred@1518530251 after retry RPC after error@585756837) -> (continuation of Deferred@1270531867 after retry RPC after error@1566691457), errback=(continuation of Deferred@1405902449 after retry RPC after error@173232753) -> (continuation of Deferred@213191538 after retry RPC after error@1380224580) -> (continuation of Deferred@1674214827 after retry RPC after error@1931111289) -> (continuation of Deferred@1464865011 after retry RPC after error@563122931) -> (continuation of Deferred@294097999 after retry RPC after error@828798117) -> (continuation of Deferred@1726365170 after retry RPC after error@1128541399) -> (continuation of Deferred@1850535196 after retry RPC after error@1812823852) -> (continuation of Deferred@1003734770 after retry RPC after error@1502836683) -> (continuation of Deferred@1354651112 after retry RPC after error@1473098956) -> (continuation of Deferred@1921784977 after retry RPC after error@811798564) -> (continuation of Deferred@315622101 after retry RPC after error@671203223) -> (continuation of Deferred@981161618 after retry RPC after error@305238985) -> (continuation of Deferred@6684144 after retry RPC after error@624760684) -> (continuation of Deferred@1167202975 after retry RPC after error@1537435199) -> (continuation of Deferred@1954974406 after retry RPC after error@969766221) -> (continuation of Deferred@1818150409 after retry RPC after error@739034915) -> (continuation of Deferred@1822210270 after retry RPC after error@1042931485) -> (continuation of Deferred@618477663 after retry RPC after error@1847772728) -> (continuation of Deferred@1735514474 after retry RPC after error@399520686) -> (continuation of Deferred@1518530251 after retry RPC after error@585756837) -> (continuation of Deferred@1270531867 after retry RPC after error@1566691457)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331) ~[main/:?]
at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:475) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:252) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.cleanup(Connection.java:737) ~[main/:?]
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:557) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.access$200(DefaultPromise.java:35) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise$1.run(DefaultPromise.java:503) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:174) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:167) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:470) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:566) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.19.101.60:35091,127.19.101.61:33871,127.19.101.62:46347) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331) ~[main/:?]
at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:475) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:252) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.cleanup(Connection.java:737) ~[main/:?]
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:557) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.access$200(DefaultPromise.java:35) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise$1.run(DefaultPromise.java:503) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:174) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:167) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:470) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:566) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:557) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.access$200(DefaultPromise.java:35) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise$1.run(DefaultPromise.java:503) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:174) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:167) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:470) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:566) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.19.101.62:46347
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=22, TimeoutTracker(timeout=30000, elapsed=29027), Traces: [0ms] refreshing cache from master, [45ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.101.60:35091, [132ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.101.61:33871, [133ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.101.62:46347, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.101.61:33871: Network error: Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871, [235ms] Sub RPC ConnectToMaster: received response from server master-127.19.101.62:46347: Network error: Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347, [496ms] Sub RPC ConnectToMaster: received response from server master-127.19.101.60:35091: OK, [500ms] delaying RPC due to: Service unavailable: Master config (127.19.101.60:35091,127.19.101.61:33871,127.19.101.62:46347) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347, [516ms] refreshing cache from master, [516ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.101.60:35091, [519ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.101.61:33871, [522ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.101.62:46347, [523ms] Sub RPC ConnectToMaster: received response from server master-127.19.101.61:33871: Network error: Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871, [525ms] Sub RPC ConnectToMaster: received response from server master-127.19.101.60:35091: OK, [527ms] Sub RPC ConnectToMaster: received response from server master-127.19.101.62:46347: Network error: Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347, [529ms] delaying RPC due to: Service unavailable: Master config (127.19.101.60:35091,127.19.101.61:33871,127.19.101.62:46347) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347, [535ms] refreshing cache from master, [535ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.101.60:35091, [535ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.101.61:33871, [536ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.101.62:46347, [538ms] Sub RPC ConnectToMaster: received response from server master-127.19.101.61:33871: Network error: Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871, [539ms] Sub RPC ConnectToMaster: received response from server master-127.19.101.60:35091: OK, [540ms] Sub RPC ConnectToMaster: received response from server master-127.19.101.62:46347: Network error: Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347, [541ms] delaying RPC due to: Service unavailable: Master config (127.19.101.60:35091,127.19.101.61:33871,127.19.101.62:46347) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347, [555ms] refreshing cache from master, [556ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.101.60:35091, [556ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.101.61:33871, [557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.101.62:46347, [560ms] Sub RPC ConnectToMaster: received response from server master-127.19.101.60:35091: OK, [560ms] Sub RPC ConnectToMaster: received response from server master-127.19.101.61:33871: Network error: Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871, [562ms] Sub RPC ConnectToMaster: received response from server master-127.19.101.62:46347: Network error: Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347, [564ms] delaying RPC due to: Service unavailable: Master config (127.19.101.60:35091,127.19.101.61:33871,127.19.101.62:46347) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347, [576ms] refreshing cache from master, [576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.101.60:35091, [576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.101.61:33871, [577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.101.62:46347, [580ms] Sub RPC ConnectToMaster: received response from server master-127.19.101.61:33871: Network error: Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871, [582ms] Sub RPC ConnectToMaster: received response from server master-127.19.101.60:35091: OK, [583ms] Sub RPC ConnectToMaster: received response from server master-127.19.101.62:46347: Network error: Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347, [584ms] delaying RPC due to: Service unavailable: Master config (127.19.101.60:35091,127.19.101.61:33871,127.19.101.62:46347) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347, [596ms] refreshing cache from master, [596ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.101.60:35091, [596ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.101.61:33871, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.101.62:46347, [602ms] Sub RPC ConnectToMaster: received response from server master-127.19.101.61:33871: Network error: Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871, [603ms] Sub RPC ConnectToMaster: received response from server master-127.19.101.60:35091: OK, [609ms] Sub RPC ConnectToMaster: received response from server master-127.19.101.62:46347: Network error: Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347, [611ms] delaying RPC due to: Service unavailable: Master config (127.19.101.60:35091,127.19.101.61:33871,127.19.101.62:46347) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347, [675ms] refreshing cache from master, [675ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.101.60:35091, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.101.61:33871, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.101.62:46347, [679ms] Sub RPC ConnectToMaster: received response from server master-127.19.101.61:33871: Network error: Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871, [681ms] Sub RPC ConnectToMaster: received response from server master-127.19.101.60:35091: OK, [682ms] Sub RPC ConnectToMaster: received response from server master-127.19.101.62:46347: Network error: Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347, [683ms] delaying RPC due to: Service unavailable: Master config (127.19.101.60:35091,127.19.101.61:33871,127.19.101.62:46347) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347, [755ms] refreshing cache from master, [756ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.101.60:35091, [756ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.101.61:33871, [758ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.101.62:46347, [759ms] Sub RPC ConnectToMaster: received response from server master-127.19.101.61:33871: Network error: Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871, [760ms] Sub RPC ConnectToMaster: received response from server master-127.19.101.60:35091: OK, [762ms] Sub RPC ConnectToMaster: received response from server master-127.19.101.62:46347: Network error: Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347, [763ms] delaying RPC due to: Service unavailable: Master config (127.19.101.60:35091,127.19.101.61:33871,127.19.101.62:46347) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347, [1015ms] refreshing cache from master, [1016ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.101.60:35091, [1016ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.101.61:33871, [1017ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.101.62:46347, [1019ms] Sub RPC ConnectToMaster: received response from server master-127.19.101.61:33871: Network error: Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871, [1020ms] Sub RPC ConnectToMaster: received response from server master-127.19.101.60:35091: OK, [1021ms] Sub RPC ConnectToMaster: received response from server master-127.19.101.62:46347: Network error: Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347, [1022ms] delaying RPC due to: Service unavailable: Master config (127.19.101.60:35091,127.19.101.61:33871,127.19.101.62:46347) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347, [1495ms] refreshing cache from master, [1496ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.101.60:35091, [1496ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.101.61:33871, [1498ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.101.62:46347, [1500ms] Sub RPC ConnectToMaster: received response from server master-127.19.101.61:33871: Network error: Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871, [1501ms] Sub RPC ConnectToMaster: received response from server master-127.19.101.60:35091: OK, [1502ms] Sub RPC ConnectToMaster: received response from server master-127.19.101.62:46347: Network error: Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347, [1503ms] delaying RPC due to: Service unavailable: Master config (127.19.101.60:35091,127.19.101.61:33871,127.19.101.62:46347) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347, [2215ms] refreshing cache from master, [2216ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.101.60:35091, [2216ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.101.61:33871, [2218ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.101.62:46347, [2219ms] Sub RPC ConnectToMaster: received response from server master-127.19.101.62:46347: Network error: Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347, [2219ms] Sub RPC ConnectToMaster: received response from server master-127.19.101.61:33871: Network error: Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871, [2222ms] Sub RPC ConnectToMaster: received response from server master-127.19.101.60:35091: OK, [2222ms] delaying RPC due to: Service unavailable: Master config (127.19.101.60:35091,127.19.101.61:33871,127.19.101.62:46347) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871, [3536ms] refreshing cache from master, [3536ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.101.60:35091, [3536ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.101.61:33871, [3538ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.101.62:46347, [3540ms] Sub RPC ConnectToMaster: received response from server master-127.19.101.60:35091: OK, [3540ms] Sub RPC ConnectToMaster: received response from server master-127.19.101.61:33871: Network error: Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871, [3541ms] Sub RPC ConnectToMaster: received response from server master-127.19.101.62:46347: Network error: Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347, [3542ms] delaying RPC due to: Service unavailable: Master config (127.19.101.60:35091,127.19.101.61:33871,127.19.101.62:46347) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347, [5555ms] refreshing cache from master, [5555ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.101.60:35091, [5555ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.101.61:33871, [5557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.101.62:46347, [5559ms] too many traces: truncated at 100 traces, deferred=Deferred@1639445259(state=PENDING, result=null, callback=(continuation of Deferred@1405902449 after retry RPC after error@173232753) -> (continuation of Deferred@213191538 after retry RPC after error@1380224580) -> (continuation of Deferred@1674214827 after retry RPC after error@1931111289) -> (continuation of Deferred@1464865011 after retry RPC after error@563122931) -> (continuation of Deferred@294097999 after retry RPC after error@828798117) -> (continuation of Deferred@1726365170 after retry RPC after error@1128541399) -> (continuation of Deferred@1850535196 after retry RPC after error@1812823852) -> (continuation of Deferred@1003734770 after retry RPC after error@1502836683) -> (continuation of Deferred@1354651112 after retry RPC after error@1473098956) -> (continuation of Deferred@1921784977 after retry RPC after error@811798564) -> (continuation of Deferred@315622101 after retry RPC after error@671203223) -> (continuation of Deferred@981161618 after retry RPC after error@305238985) -> (continuation of Deferred@6684144 after retry RPC after error@624760684) -> (continuation of Deferred@1167202975 after retry RPC after error@1537435199) -> (continuation of Deferred@1954974406 after retry RPC after error@969766221) -> (continuation of Deferred@1818150409 after retry RPC after error@739034915) -> (continuation of Deferred@1822210270 after retry RPC after error@1042931485) -> (continuation of Deferred@618477663 after retry RPC after error@1847772728) -> (continuation of Deferred@1735514474 after retry RPC after error@399520686) -> (continuation of Deferred@1518530251 after retry RPC after error@585756837) -> (continuation of Deferred@1270531867 after retry RPC after error@1566691457), errback=(continuation of Deferred@1405902449 after retry RPC after error@173232753) -> (continuation of Deferred@213191538 after retry RPC after error@1380224580) -> (continuation of Deferred@1674214827 after retry RPC after error@1931111289) -> (continuation of Deferred@1464865011 after retry RPC after error@563122931) -> (continuation of Deferred@294097999 after retry RPC after error@828798117) -> (continuation of Deferred@1726365170 after retry RPC after error@1128541399) -> (continuation of Deferred@1850535196 after retry RPC after error@1812823852) -> (continuation of Deferred@1003734770 after retry RPC after error@1502836683) -> (continuation of Deferred@1354651112 after retry RPC after error@1473098956) -> (continuation of Deferred@1921784977 after retry RPC after error@811798564) -> (continuation of Deferred@315622101 after retry RPC after error@671203223) -> (continuation of Deferred@981161618 after retry RPC after error@305238985) -> (continuation of Deferred@6684144 after retry RPC after error@624760684) -> (continuation of Deferred@1167202975 after retry RPC after error@1537435199) -> (continuation of Deferred@1954974406 after retry RPC after error@969766221) -> (continuation of Deferred@1818150409 after retry RPC after error@739034915) -> (continuation of Deferred@1822210270 after retry RPC after error@1042931485) -> (continuation of Deferred@618477663 after retry RPC after error@1847772728) -> (continuation of Deferred@1735514474 after retry RPC after error@399520686) -> (continuation of Deferred@1518530251 after retry RPC after error@585756837) -> (continuation of Deferred@1270531867 after retry RPC after error@1566691457)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331)
at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:475)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:252)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:557)
at io.netty.util.concurrent.DefaultPromise.access$200(DefaultPromise.java:35)
at io.netty.util.concurrent.DefaultPromise$1.run(DefaultPromise.java:503)
at io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:174)
at io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:167)
at io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:470)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:566)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.19.101.60:35091,127.19.101.61:33871,127.19.101.62:46347) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.101.61:33871(127.19.101.61:33871): Connection refused: /127.19.101.61:33871,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331)
at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:475)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:252)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:557)
at io.netty.util.concurrent.DefaultPromise.access$200(DefaultPromise.java:35)
at io.netty.util.concurrent.DefaultPromise$1.run(DefaultPromise.java:503)
at io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:174)
at io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:167)
at io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:470)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:566)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.101.62:46347(127.19.101.62:46347): Connection refused: /127.19.101.62:46347
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
... 14 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.19.101.62:46347
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)