Note: This is test shard 4 of 8.
[==========] Running 3 tests from 3 test suites.
[----------] Global test environment set-up.
[----------] 1 test from AllTypesItest/1, where TypeParam = kudu::client::IntKeysTestSetup<kudu::client::KeyTypeWrapper<(kudu::DataType)3> >
[ RUN ] AllTypesItest/1.TestTimestampPadding
WARNING: Logging before InitGoogleLogging() is written to STDERR
I20260810 22:28:34.366389 8266 test_util.cc:268] Using random seed: -538091644
2026-08-10T22:28:34Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
2026-08-10T22:28:34Z Disabled control of system clock
I20260810 22:28:34.462057 8266 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskMIzrRq/build/debug/bin/kudu
/tmp/dist-test-taskMIzrRq/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/wal
--fs_data_dirs=/tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/logs
--server_dump_info_path=/tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
master
run
--ipki_ca_key_size=768
--tsk_num_rsa_bits=512
--rpc_bind_addresses=127.8.18.190:40323
--webserver_interface=127.8.18.190
--webserver_port=0
--builtin_ntp_servers=127.8.18.148:42749
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--rpc_reuseport=true
--master_addresses=127.8.18.190:40323 with env {}
W20260810 22:28:34.590867 8278 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20260810 22:28:34.591154 8278 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20260810 22:28:34.591189 8278 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20260810 22:28:34.595176 8278 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
W20260810 22:28:34.595230 8278 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20260810 22:28:34.595248 8278 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
W20260810 22:28:34.595264 8278 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
I20260810 22:28:34.600224 8278 master_runner.cc:386] Master server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.8.18.148:42749
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/data
--fs_wal_dir=/tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/wal
--ipki_ca_key_size=768
--master_addresses=127.8.18.190:40323
--ipki_server_key_size=768
--openssl_security_level_override=0
--tsk_num_rsa_bits=512
--rpc_bind_addresses=127.8.18.190:40323
--rpc_reuseport=true
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/data/info.pb
--webserver_interface=127.8.18.190
--webserver_port=0
--never_fsync=true
--heap_profile_path=/tmp/kudu.8278
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--log_dir=/tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/logs
--logbuflevel=-1
--logtostderr=true
Master server version:
kudu 1.18.0-SNAPSHOT
revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
build type DEBUG
built by systest at 10 Aug 2026 21:55:49 UTC on ccycloud.quasar-ktihrd.root.comops.site
build id 4579236
I20260810 22:28:34.601794 8278 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20260810 22:28:34.604732 8278 file_cache.cc:493] Constructed file cache file cache with capacity 419430
W20260810 22:28:34.618594 8290 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20260810 22:28:34.619028 8285 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20260810 22:28:34.620694 8287 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20260810 22:28:34.621110 8278 server_base.cc:1034] running on GCE node
I20260810 22:28:34.622013 8278 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20260810 22:28:34.624058 8278 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20260810 22:28:34.625267 8278 hybrid_clock.cc:648] HybridClock initialized: now 1786400914625226 us; error 61 us; skew 500 ppm
I20260810 22:28:34.628561 8278 webserver.cc:460] Webserver started at http://127.8.18.190:45723/ using document root <none> and password file <none>
I20260810 22:28:34.629647 8278 fs_manager.cc:359] Metadata directory not provided
I20260810 22:28:34.629762 8278 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20260810 22:28:34.630214 8278 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20260810 22:28:34.632994 8278 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/data/instance:
uuid: "e52d1e743df1434f88d46b1016be2ae7"
format_stamp: "Formatted at 2026-08-10 22:28:34 on dist-test-slave-bw88"
I20260810 22:28:34.633821 8278 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/wal/instance:
uuid: "e52d1e743df1434f88d46b1016be2ae7"
format_stamp: "Formatted at 2026-08-10 22:28:34 on dist-test-slave-bw88"
I20260810 22:28:34.639377 8278 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.004s sys 0.000s
I20260810 22:28:34.643167 8298 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20260810 22:28:34.644950 8278 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.004s sys 0.000s
I20260810 22:28:34.645187 8278 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/data,/tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/wal
uuid: "e52d1e743df1434f88d46b1016be2ae7"
format_stamp: "Formatted at 2026-08-10 22:28:34 on dist-test-slave-bw88"
I20260810 22:28:34.645370 8278 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/wal
metadata directory: /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/wal
1 data directories: /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20260810 22:28:34.659544 8278 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
I20260810 22:28:34.660444 8278 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20260810 22:28:34.660710 8278 kserver.cc:163] Server-wide thread pool size limit: 3276
I20260810 22:28:34.671517 8278 rpc_server.cc:307] RPC server started. Bound to: 127.8.18.190:40323
I20260810 22:28:34.671589 8366 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.8.18.190:40323 every 8 connection(s)
I20260810 22:28:34.672953 8278 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/data/info.pb
I20260810 22:28:34.680142 8266 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskMIzrRq/build/debug/bin/kudu as pid 8278
I20260810 22:28:34.680403 8266 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/wal/instance
I20260810 22:28:34.681082 8367 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
I20260810 22:28:34.692410 8367 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P e52d1e743df1434f88d46b1016be2ae7: Bootstrap starting.
I20260810 22:28:34.697899 8367 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P e52d1e743df1434f88d46b1016be2ae7: Neither blocks nor log segments found. Creating new log.
I20260810 22:28:34.705230 8367 log.cc:824] T 00000000000000000000000000000000 P e52d1e743df1434f88d46b1016be2ae7: Log is configured to *not* fsync() on all Append() calls
I20260810 22:28:34.708684 8367 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P e52d1e743df1434f88d46b1016be2ae7: No bootstrap required, opened a new log
I20260810 22:28:34.714011 8367 raft_consensus.cc:348] T 00000000000000000000000000000000 P e52d1e743df1434f88d46b1016be2ae7 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "e52d1e743df1434f88d46b1016be2ae7" member_type: VOTER last_known_addr { host: "127.8.18.190" port: 40323 } }
I20260810 22:28:34.714316 8367 raft_consensus.cc:374] T 00000000000000000000000000000000 P e52d1e743df1434f88d46b1016be2ae7 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20260810 22:28:34.714432 8367 raft_consensus.cc:729] T 00000000000000000000000000000000 P e52d1e743df1434f88d46b1016be2ae7 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e52d1e743df1434f88d46b1016be2ae7, State: Initialized, Role: FOLLOWER
I20260810 22:28:34.715139 8367 consensus_queue.cc:260] T 00000000000000000000000000000000 P e52d1e743df1434f88d46b1016be2ae7 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "e52d1e743df1434f88d46b1016be2ae7" member_type: VOTER last_known_addr { host: "127.8.18.190" port: 40323 } }
I20260810 22:28:34.715350 8367 raft_consensus.cc:388] T 00000000000000000000000000000000 P e52d1e743df1434f88d46b1016be2ae7 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20260810 22:28:34.715494 8367 raft_consensus.cc:482] T 00000000000000000000000000000000 P e52d1e743df1434f88d46b1016be2ae7 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20260810 22:28:34.715659 8367 raft_consensus.cc:3037] T 00000000000000000000000000000000 P e52d1e743df1434f88d46b1016be2ae7 [term 0 FOLLOWER]: Advancing to term 1
I20260810 22:28:34.716611 8367 raft_consensus.cc:504] T 00000000000000000000000000000000 P e52d1e743df1434f88d46b1016be2ae7 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "e52d1e743df1434f88d46b1016be2ae7" member_type: VOTER last_known_addr { host: "127.8.18.190" port: 40323 } }
I20260810 22:28:34.717186 8367 leader_election.cc:302] T 00000000000000000000000000000000 P e52d1e743df1434f88d46b1016be2ae7 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: e52d1e743df1434f88d46b1016be2ae7; no voters:
I20260810 22:28:34.717638 8367 leader_election.cc:288] T 00000000000000000000000000000000 P e52d1e743df1434f88d46b1016be2ae7 [CANDIDATE]: Term 1 election: Requested vote from peers
I20260810 22:28:34.717806 8374 raft_consensus.cc:2781] T 00000000000000000000000000000000 P e52d1e743df1434f88d46b1016be2ae7 [term 1 FOLLOWER]: Leader election won for term 1
I20260810 22:28:34.718092 8374 raft_consensus.cc:686] T 00000000000000000000000000000000 P e52d1e743df1434f88d46b1016be2ae7 [term 1 LEADER]: Becoming Leader. State: Replica: e52d1e743df1434f88d46b1016be2ae7, State: Running, Role: LEADER
I20260810 22:28:34.718528 8374 consensus_queue.cc:237] T 00000000000000000000000000000000 P e52d1e743df1434f88d46b1016be2ae7 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "e52d1e743df1434f88d46b1016be2ae7" member_type: VOTER last_known_addr { host: "127.8.18.190" port: 40323 } }
I20260810 22:28:34.719337 8367 sys_catalog.cc:564] T 00000000000000000000000000000000 P e52d1e743df1434f88d46b1016be2ae7 [sys.catalog]: configured and running, proceeding with master startup.
I20260810 22:28:34.721323 8377 sys_catalog.cc:455] T 00000000000000000000000000000000 P e52d1e743df1434f88d46b1016be2ae7 [sys.catalog]: SysCatalogTable state changed. Reason: New leader e52d1e743df1434f88d46b1016be2ae7. Latest consensus state: current_term: 1 leader_uuid: "e52d1e743df1434f88d46b1016be2ae7" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "e52d1e743df1434f88d46b1016be2ae7" member_type: VOTER last_known_addr { host: "127.8.18.190" port: 40323 } } }
I20260810 22:28:34.721410 8376 sys_catalog.cc:455] T 00000000000000000000000000000000 P e52d1e743df1434f88d46b1016be2ae7 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 leader_uuid: "e52d1e743df1434f88d46b1016be2ae7" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "e52d1e743df1434f88d46b1016be2ae7" member_type: VOTER last_known_addr { host: "127.8.18.190" port: 40323 } } }
I20260810 22:28:34.721536 8377 sys_catalog.cc:458] T 00000000000000000000000000000000 P e52d1e743df1434f88d46b1016be2ae7 [sys.catalog]: This master's current role is: LEADER
I20260810 22:28:34.721537 8376 sys_catalog.cc:458] T 00000000000000000000000000000000 P e52d1e743df1434f88d46b1016be2ae7 [sys.catalog]: This master's current role is: LEADER
W20260810 22:28:34.726651 8393 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e52d1e743df1434f88d46b1016be2ae7: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
W20260810 22:28:34.726751 8393 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
I20260810 22:28:34.726847 8390 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
I20260810 22:28:34.727764 8390 catalog_manager.cc:1484] Initializing Kudu cluster ID...
I20260810 22:28:34.737941 8390 catalog_manager.cc:1347] Generated new cluster ID: 9263430df01e4fe1b9447e2b34868383
I20260810 22:28:34.738040 8390 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
I20260810 22:28:34.767127 8390 catalog_manager.cc:1370] Generated new certificate authority record
I20260810 22:28:34.768237 8390 catalog_manager.cc:1504] Loading token signing keys...
I20260810 22:28:34.779757 8390 catalog_manager.cc:5868] T 00000000000000000000000000000000 P e52d1e743df1434f88d46b1016be2ae7: Generated new TSK 0
I20260810 22:28:34.780709 8390 catalog_manager.cc:1514] Initializing in-progress tserver states...
I20260810 22:28:34.788324 8266 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskMIzrRq/build/debug/bin/kudu
/tmp/dist-test-taskMIzrRq/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-0/wal
--fs_data_dirs=/tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-0/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-0/logs
--server_dump_info_path=/tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-0/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.8.18.129:0
--local_ip_for_outbound_sockets=127.8.18.129
--webserver_interface=127.8.18.129
--webserver_port=0
--tserver_master_addrs=127.8.18.190:40323
--builtin_ntp_servers=127.8.18.148:42749
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--flush_threshold_mb=1
--tablet_delta_store_major_compact_min_ratio=0.001
--consensus_max_batch_size_bytes=2097152 with env {}
W20260810 22:28:34.943596 8398 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20260810 22:28:34.943912 8398 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20260810 22:28:34.943955 8398 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20260810 22:28:34.948302 8398 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20260810 22:28:34.948375 8398 flags.cc:425] Enabled experimental flag: --tablet_delta_store_major_compact_min_ratio=0.001
W20260810 22:28:34.948391 8398 flags.cc:425] Enabled experimental flag: --flush_threshold_mb=1
W20260810 22:28:34.948482 8398 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.8.18.129
I20260810 22:28:34.954370 8398 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.8.18.148:42749
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--consensus_max_batch_size_bytes=2097152
--fs_data_dirs=/tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-0/data
--fs_wal_dir=/tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-0/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.8.18.129:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-0/data/info.pb
--webserver_interface=127.8.18.129
--webserver_port=0
--tablet_delta_store_major_compact_min_ratio=0.001
--flush_threshold_mb=1
--tserver_master_addrs=127.8.18.190:40323
--never_fsync=true
--heap_profile_path=/tmp/kudu.8398
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.8.18.129
--log_dir=/tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-0/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
build type DEBUG
built by systest at 10 Aug 2026 21:55:49 UTC on ccycloud.quasar-ktihrd.root.comops.site
build id 4579236
I20260810 22:28:34.955668 8398 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20260810 22:28:34.957268 8398 file_cache.cc:493] Constructed file cache file cache with capacity 419430
W20260810 22:28:34.969233 8409 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20260810 22:28:34.969435 8398 server_base.cc:1034] running on GCE node
W20260810 22:28:34.969259 8411 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20260810 22:28:34.969259 8407 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20260810 22:28:34.969986 8398 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20260810 22:28:34.970687 8398 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20260810 22:28:34.971882 8398 hybrid_clock.cc:648] HybridClock initialized: now 1786400914971870 us; error 37 us; skew 500 ppm
I20260810 22:28:34.978294 8398 webserver.cc:460] Webserver started at http://127.8.18.129:45343/ using document root <none> and password file <none>
I20260810 22:28:34.978958 8398 fs_manager.cc:359] Metadata directory not provided
I20260810 22:28:34.979059 8398 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20260810 22:28:34.979324 8398 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20260810 22:28:34.981482 8398 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-0/data/instance:
uuid: "747982f4b834445b93f077dd1295b2ff"
format_stamp: "Formatted at 2026-08-10 22:28:34 on dist-test-slave-bw88"
I20260810 22:28:34.982086 8398 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-0/wal/instance:
uuid: "747982f4b834445b93f077dd1295b2ff"
format_stamp: "Formatted at 2026-08-10 22:28:34 on dist-test-slave-bw88"
I20260810 22:28:34.986229 8398 fs_manager.cc:691] Time spent creating directory manager: real 0.004s user 0.004s sys 0.000s
I20260810 22:28:34.989064 8418 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20260810 22:28:34.990482 8398 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.002s sys 0.001s
I20260810 22:28:34.990631 8398 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-0/data,/tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-0/wal
uuid: "747982f4b834445b93f077dd1295b2ff"
format_stamp: "Formatted at 2026-08-10 22:28:34 on dist-test-slave-bw88"
I20260810 22:28:34.990751 8398 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-0/wal
metadata directory: /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-0/wal
1 data directories: /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-0/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20260810 22:28:35.013718 8398 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
I20260810 22:28:35.014854 8398 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20260810 22:28:35.015120 8398 kserver.cc:163] Server-wide thread pool size limit: 3276
I20260810 22:28:35.016072 8398 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20260810 22:28:35.017679 8398 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20260810 22:28:35.017762 8398 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20260810 22:28:35.017845 8398 ts_tablet_manager.cc:610] Registered 0 tablets
I20260810 22:28:35.017899 8398 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20260810 22:28:35.035611 8398 rpc_server.cc:307] RPC server started. Bound to: 127.8.18.129:40911
I20260810 22:28:35.035677 8578 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.8.18.129:40911 every 8 connection(s)
I20260810 22:28:35.036733 8398 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-0/data/info.pb
I20260810 22:28:35.037853 8266 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskMIzrRq/build/debug/bin/kudu as pid 8398
I20260810 22:28:35.037971 8266 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-0/wal/instance
I20260810 22:28:35.040699 8266 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskMIzrRq/build/debug/bin/kudu
/tmp/dist-test-taskMIzrRq/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-1/wal
--fs_data_dirs=/tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-1/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-1/logs
--server_dump_info_path=/tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-1/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.8.18.130:0
--local_ip_for_outbound_sockets=127.8.18.130
--webserver_interface=127.8.18.130
--webserver_port=0
--tserver_master_addrs=127.8.18.190:40323
--builtin_ntp_servers=127.8.18.148:42749
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--flush_threshold_mb=1
--tablet_delta_store_major_compact_min_ratio=0.001
--consensus_max_batch_size_bytes=2097152 with env {}
I20260810 22:28:35.051890 8579 heartbeater.cc:347] Connected to a master server at 127.8.18.190:40323
I20260810 22:28:35.052194 8579 heartbeater.cc:464] Registering TS with master...
I20260810 22:28:35.052794 8579 heartbeater.cc:511] Master 127.8.18.190:40323 requested a full tablet report, sending...
I20260810 22:28:35.055505 8319 ts_manager.cc:194] Registered new tserver with Master: 747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911)
I20260810 22:28:35.058248 8319 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.8.18.129:59931
W20260810 22:28:35.206775 8582 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20260810 22:28:35.207055 8582 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20260810 22:28:35.207099 8582 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20260810 22:28:35.211236 8582 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20260810 22:28:35.211304 8582 flags.cc:425] Enabled experimental flag: --tablet_delta_store_major_compact_min_ratio=0.001
W20260810 22:28:35.211320 8582 flags.cc:425] Enabled experimental flag: --flush_threshold_mb=1
W20260810 22:28:35.211440 8582 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.8.18.130
I20260810 22:28:35.216377 8582 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.8.18.148:42749
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--consensus_max_batch_size_bytes=2097152
--fs_data_dirs=/tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-1/data
--fs_wal_dir=/tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-1/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.8.18.130:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-1/data/info.pb
--webserver_interface=127.8.18.130
--webserver_port=0
--tablet_delta_store_major_compact_min_ratio=0.001
--flush_threshold_mb=1
--tserver_master_addrs=127.8.18.190:40323
--never_fsync=true
--heap_profile_path=/tmp/kudu.8582
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.8.18.130
--log_dir=/tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-1/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
build type DEBUG
built by systest at 10 Aug 2026 21:55:49 UTC on ccycloud.quasar-ktihrd.root.comops.site
build id 4579236
I20260810 22:28:35.217656 8582 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20260810 22:28:35.219184 8582 file_cache.cc:493] Constructed file cache file cache with capacity 419430
W20260810 22:28:35.230769 8594 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20260810 22:28:35.230801 8591 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20260810 22:28:35.230739 8589 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20260810 22:28:35.230810 8582 server_base.cc:1034] running on GCE node
I20260810 22:28:35.231479 8582 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20260810 22:28:35.232195 8582 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20260810 22:28:35.233419 8582 hybrid_clock.cc:648] HybridClock initialized: now 1786400915233400 us; error 36 us; skew 500 ppm
I20260810 22:28:35.235981 8582 webserver.cc:460] Webserver started at http://127.8.18.130:44273/ using document root <none> and password file <none>
I20260810 22:28:35.236613 8582 fs_manager.cc:359] Metadata directory not provided
I20260810 22:28:35.236713 8582 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20260810 22:28:35.236974 8582 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20260810 22:28:35.239269 8582 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-1/data/instance:
uuid: "01cff8a283e94658aea5fad9fa0b2db4"
format_stamp: "Formatted at 2026-08-10 22:28:35 on dist-test-slave-bw88"
I20260810 22:28:35.239907 8582 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-1/wal/instance:
uuid: "01cff8a283e94658aea5fad9fa0b2db4"
format_stamp: "Formatted at 2026-08-10 22:28:35 on dist-test-slave-bw88"
I20260810 22:28:35.244092 8582 fs_manager.cc:691] Time spent creating directory manager: real 0.004s user 0.006s sys 0.000s
I20260810 22:28:35.246909 8603 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20260810 22:28:35.248286 8582 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.001s sys 0.000s
I20260810 22:28:35.248431 8582 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-1/data,/tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-1/wal
uuid: "01cff8a283e94658aea5fad9fa0b2db4"
format_stamp: "Formatted at 2026-08-10 22:28:35 on dist-test-slave-bw88"
I20260810 22:28:35.248548 8582 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-1/wal
metadata directory: /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-1/wal
1 data directories: /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-1/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20260810 22:28:35.269280 8582 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
I20260810 22:28:35.270510 8582 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20260810 22:28:35.270780 8582 kserver.cc:163] Server-wide thread pool size limit: 3276
I20260810 22:28:35.271601 8582 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20260810 22:28:35.272771 8582 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20260810 22:28:35.272848 8582 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20260810 22:28:35.272921 8582 ts_tablet_manager.cc:610] Registered 0 tablets
I20260810 22:28:35.272979 8582 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20260810 22:28:35.292274 8582 rpc_server.cc:307] RPC server started. Bound to: 127.8.18.130:33631
I20260810 22:28:35.292368 8755 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.8.18.130:33631 every 8 connection(s)
I20260810 22:28:35.293486 8582 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-1/data/info.pb
I20260810 22:28:35.300829 8266 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskMIzrRq/build/debug/bin/kudu as pid 8582
I20260810 22:28:35.300974 8266 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-1/wal/instance
I20260810 22:28:35.303613 8266 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskMIzrRq/build/debug/bin/kudu
/tmp/dist-test-taskMIzrRq/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-2/wal
--fs_data_dirs=/tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-2/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-2/logs
--server_dump_info_path=/tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-2/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.8.18.131:0
--local_ip_for_outbound_sockets=127.8.18.131
--webserver_interface=127.8.18.131
--webserver_port=0
--tserver_master_addrs=127.8.18.190:40323
--builtin_ntp_servers=127.8.18.148:42749
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--flush_threshold_mb=1
--tablet_delta_store_major_compact_min_ratio=0.001
--consensus_max_batch_size_bytes=2097152 with env {}
I20260810 22:28:35.306569 8756 heartbeater.cc:347] Connected to a master server at 127.8.18.190:40323
I20260810 22:28:35.306869 8756 heartbeater.cc:464] Registering TS with master...
I20260810 22:28:35.307605 8756 heartbeater.cc:511] Master 127.8.18.190:40323 requested a full tablet report, sending...
I20260810 22:28:35.308853 8319 ts_manager.cc:194] Registered new tserver with Master: 01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631)
I20260810 22:28:35.309517 8319 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.8.18.130:56349
W20260810 22:28:35.453352 8762 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20260810 22:28:35.453755 8762 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20260810 22:28:35.453836 8762 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20260810 22:28:35.458057 8762 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20260810 22:28:35.458134 8762 flags.cc:425] Enabled experimental flag: --tablet_delta_store_major_compact_min_ratio=0.001
W20260810 22:28:35.458176 8762 flags.cc:425] Enabled experimental flag: --flush_threshold_mb=1
W20260810 22:28:35.458256 8762 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.8.18.131
I20260810 22:28:35.463373 8762 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.8.18.148:42749
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--consensus_max_batch_size_bytes=2097152
--fs_data_dirs=/tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-2/data
--fs_wal_dir=/tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-2/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.8.18.131:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-2/data/info.pb
--webserver_interface=127.8.18.131
--webserver_port=0
--tablet_delta_store_major_compact_min_ratio=0.001
--flush_threshold_mb=1
--tserver_master_addrs=127.8.18.190:40323
--never_fsync=true
--heap_profile_path=/tmp/kudu.8762
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.8.18.131
--log_dir=/tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-2/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
build type DEBUG
built by systest at 10 Aug 2026 21:55:49 UTC on ccycloud.quasar-ktihrd.root.comops.site
build id 4579236
I20260810 22:28:35.464627 8762 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20260810 22:28:35.466208 8762 file_cache.cc:493] Constructed file cache file cache with capacity 419430
W20260810 22:28:35.478166 8770 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20260810 22:28:35.478188 8767 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20260810 22:28:35.478152 8768 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20260810 22:28:35.478515 8762 server_base.cc:1034] running on GCE node
I20260810 22:28:35.479017 8762 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20260810 22:28:35.479686 8762 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20260810 22:28:35.480880 8762 hybrid_clock.cc:648] HybridClock initialized: now 1786400915480843 us; error 49 us; skew 500 ppm
I20260810 22:28:35.483080 8762 webserver.cc:460] Webserver started at http://127.8.18.131:45679/ using document root <none> and password file <none>
I20260810 22:28:35.483706 8762 fs_manager.cc:359] Metadata directory not provided
I20260810 22:28:35.483805 8762 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20260810 22:28:35.484051 8762 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20260810 22:28:35.486227 8762 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-2/data/instance:
uuid: "d06d8f442c824758ae944ffe71e3cb44"
format_stamp: "Formatted at 2026-08-10 22:28:35 on dist-test-slave-bw88"
I20260810 22:28:35.487790 8762 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-2/wal/instance:
uuid: "d06d8f442c824758ae944ffe71e3cb44"
format_stamp: "Formatted at 2026-08-10 22:28:35 on dist-test-slave-bw88"
I20260810 22:28:35.492483 8762 fs_manager.cc:691] Time spent creating directory manager: real 0.004s user 0.000s sys 0.003s
I20260810 22:28:35.495404 8778 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20260810 22:28:35.496711 8762 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.001s sys 0.000s
I20260810 22:28:35.496855 8762 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-2/data,/tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-2/wal
uuid: "d06d8f442c824758ae944ffe71e3cb44"
format_stamp: "Formatted at 2026-08-10 22:28:35 on dist-test-slave-bw88"
I20260810 22:28:35.496970 8762 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-2/wal
metadata directory: /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-2/wal
1 data directories: /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-2/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20260810 22:28:35.512488 8762 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
I20260810 22:28:35.513558 8762 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20260810 22:28:35.513803 8762 kserver.cc:163] Server-wide thread pool size limit: 3276
I20260810 22:28:35.514524 8762 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20260810 22:28:35.515621 8762 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20260810 22:28:35.515699 8762 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20260810 22:28:35.515775 8762 ts_tablet_manager.cc:610] Registered 0 tablets
I20260810 22:28:35.515827 8762 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20260810 22:28:35.531595 8762 rpc_server.cc:307] RPC server started. Bound to: 127.8.18.131:36507
I20260810 22:28:35.531649 8932 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.8.18.131:36507 every 8 connection(s)
I20260810 22:28:35.532685 8762 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-2/data/info.pb
I20260810 22:28:35.533620 8266 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskMIzrRq/build/debug/bin/kudu as pid 8762
I20260810 22:28:35.533768 8266 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-2/wal/instance
I20260810 22:28:35.545037 8933 heartbeater.cc:347] Connected to a master server at 127.8.18.190:40323
I20260810 22:28:35.545323 8933 heartbeater.cc:464] Registering TS with master...
I20260810 22:28:35.545964 8933 heartbeater.cc:511] Master 127.8.18.190:40323 requested a full tablet report, sending...
I20260810 22:28:35.547075 8319 ts_manager.cc:194] Registered new tserver with Master: d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507)
I20260810 22:28:35.547694 8319 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.8.18.131:37617
I20260810 22:28:35.549278 8266 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
I20260810 22:28:35.567457 8319 catalog_manager.cc:1908] Servicing CreateTable request from {username='slave'} at 127.0.0.1:36492:
name: "all-types-table"
schema {
columns {
name: "key"
type: INT16
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int8_val"
type: INT8
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int16_val"
type: INT16
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int32_val"
type: INT32
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int64_val"
type: INT64
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "timestamp_val"
type: UNIXTIME_MICROS
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "date_val"
type: DATE
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "varchar_val"
type: VARCHAR
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
type_attributes {
length: 65535
}
immutable: false
}
columns {
name: "bool_val"
type: BOOL
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "float_val"
type: FLOAT
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "double_val"
type: DOUBLE
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "binary_val"
type: BINARY
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "decimal32_val"
type: DECIMAL32
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
type_attributes {
precision: 9
scale: 0
}
immutable: false
}
columns {
name: "decimal64_val"
type: DECIMAL64
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
type_attributes {
precision: 18
scale: 0
}
immutable: false
}
columns {
name: "decimal128_val"
type: DECIMAL128
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
type_attributes {
precision: 38
scale: 0
}
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
rows: "<redacted>""\004\001\000\000\000\252*\004\001\000\000\000TU"
indirect_data: "<redacted>"""
}
partition_schema {
range_schema {
columns {
name: "key"
}
}
}
W20260810 22:28:35.571692 8319 catalog_manager.cc:6854] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table all-types-table in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20260810 22:28:35.595537 8490 tablet_service.cc:1467] Processing CreateTablet for tablet 1f99182d93ed483e9f520bf312a591e3 (DEFAULT_TABLE table=all-types-table [id=05170d4d8109473db3e4ed0cea274fab]), partition=RANGE (key) PARTITION VALUES < 10922
I20260810 22:28:35.595556 8488 tablet_service.cc:1467] Processing CreateTablet for tablet 459e1d55c65e4857a7ccba6b13316243 (DEFAULT_TABLE table=all-types-table [id=05170d4d8109473db3e4ed0cea274fab]), partition=RANGE (key) PARTITION 10922 <= VALUES < 21844
I20260810 22:28:35.596944 8486 tablet_service.cc:1467] Processing CreateTablet for tablet b29f930721b943039d1df1f03e073a74 (DEFAULT_TABLE table=all-types-table [id=05170d4d8109473db3e4ed0cea274fab]), partition=RANGE (key) PARTITION 21844 <= VALUES
I20260810 22:28:35.596694 8669 tablet_service.cc:1467] Processing CreateTablet for tablet 459e1d55c65e4857a7ccba6b13316243 (DEFAULT_TABLE table=all-types-table [id=05170d4d8109473db3e4ed0cea274fab]), partition=RANGE (key) PARTITION 10922 <= VALUES < 21844
I20260810 22:28:35.596709 8670 tablet_service.cc:1467] Processing CreateTablet for tablet 1f99182d93ed483e9f520bf312a591e3 (DEFAULT_TABLE table=all-types-table [id=05170d4d8109473db3e4ed0cea274fab]), partition=RANGE (key) PARTITION VALUES < 10922
I20260810 22:28:35.596694 8668 tablet_service.cc:1467] Processing CreateTablet for tablet b29f930721b943039d1df1f03e073a74 (DEFAULT_TABLE table=all-types-table [id=05170d4d8109473db3e4ed0cea274fab]), partition=RANGE (key) PARTITION 21844 <= VALUES
I20260810 22:28:35.598768 8490 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 1f99182d93ed483e9f520bf312a591e3. 1 dirs total, 0 dirs full, 0 dirs failed
I20260810 22:28:35.598779 8669 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 459e1d55c65e4857a7ccba6b13316243. 1 dirs total, 0 dirs full, 0 dirs failed
I20260810 22:28:35.600096 8486 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet b29f930721b943039d1df1f03e073a74. 1 dirs total, 0 dirs full, 0 dirs failed
I20260810 22:28:35.599784 8843 tablet_service.cc:1467] Processing CreateTablet for tablet 459e1d55c65e4857a7ccba6b13316243 (DEFAULT_TABLE table=all-types-table [id=05170d4d8109473db3e4ed0cea274fab]), partition=RANGE (key) PARTITION 10922 <= VALUES < 21844
I20260810 22:28:35.601133 8843 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 459e1d55c65e4857a7ccba6b13316243. 1 dirs total, 0 dirs full, 0 dirs failed
I20260810 22:28:35.599586 8842 tablet_service.cc:1467] Processing CreateTablet for tablet b29f930721b943039d1df1f03e073a74 (DEFAULT_TABLE table=all-types-table [id=05170d4d8109473db3e4ed0cea274fab]), partition=RANGE (key) PARTITION 21844 <= VALUES
I20260810 22:28:35.604372 8842 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet b29f930721b943039d1df1f03e073a74. 1 dirs total, 0 dirs full, 0 dirs failed
I20260810 22:28:35.599586 8844 tablet_service.cc:1467] Processing CreateTablet for tablet 1f99182d93ed483e9f520bf312a591e3 (DEFAULT_TABLE table=all-types-table [id=05170d4d8109473db3e4ed0cea274fab]), partition=RANGE (key) PARTITION VALUES < 10922
I20260810 22:28:35.606581 8844 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 1f99182d93ed483e9f520bf312a591e3. 1 dirs total, 0 dirs full, 0 dirs failed
I20260810 22:28:35.606889 8670 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 1f99182d93ed483e9f520bf312a591e3. 1 dirs total, 0 dirs full, 0 dirs failed
I20260810 22:28:35.609429 8488 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 459e1d55c65e4857a7ccba6b13316243. 1 dirs total, 0 dirs full, 0 dirs failed
I20260810 22:28:35.609892 8668 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet b29f930721b943039d1df1f03e073a74. 1 dirs total, 0 dirs full, 0 dirs failed
I20260810 22:28:35.610643 8954 tablet_bootstrap.cc:492] T 459e1d55c65e4857a7ccba6b13316243 P d06d8f442c824758ae944ffe71e3cb44: Bootstrap starting.
I20260810 22:28:35.613709 8954 tablet_bootstrap.cc:654] T 459e1d55c65e4857a7ccba6b13316243 P d06d8f442c824758ae944ffe71e3cb44: Neither blocks nor log segments found. Creating new log.
I20260810 22:28:35.613870 8955 tablet_bootstrap.cc:492] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4: Bootstrap starting.
I20260810 22:28:35.614907 8953 tablet_bootstrap.cc:492] T 1f99182d93ed483e9f520bf312a591e3 P 747982f4b834445b93f077dd1295b2ff: Bootstrap starting.
I20260810 22:28:35.615626 8954 log.cc:824] T 459e1d55c65e4857a7ccba6b13316243 P d06d8f442c824758ae944ffe71e3cb44: Log is configured to *not* fsync() on all Append() calls
I20260810 22:28:35.617384 8953 tablet_bootstrap.cc:654] T 1f99182d93ed483e9f520bf312a591e3 P 747982f4b834445b93f077dd1295b2ff: Neither blocks nor log segments found. Creating new log.
I20260810 22:28:35.617465 8955 tablet_bootstrap.cc:654] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4: Neither blocks nor log segments found. Creating new log.
I20260810 22:28:35.618388 8954 tablet_bootstrap.cc:492] T 459e1d55c65e4857a7ccba6b13316243 P d06d8f442c824758ae944ffe71e3cb44: No bootstrap required, opened a new log
I20260810 22:28:35.618633 8954 ts_tablet_manager.cc:1397] T 459e1d55c65e4857a7ccba6b13316243 P d06d8f442c824758ae944ffe71e3cb44: Time spent bootstrapping tablet: real 0.008s user 0.007s sys 0.000s
I20260810 22:28:35.618611 8955 log.cc:824] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4: Log is configured to *not* fsync() on all Append() calls
I20260810 22:28:35.618853 8953 log.cc:824] T 1f99182d93ed483e9f520bf312a591e3 P 747982f4b834445b93f077dd1295b2ff: Log is configured to *not* fsync() on all Append() calls
I20260810 22:28:35.621186 8953 tablet_bootstrap.cc:492] T 1f99182d93ed483e9f520bf312a591e3 P 747982f4b834445b93f077dd1295b2ff: No bootstrap required, opened a new log
I20260810 22:28:35.621199 8955 tablet_bootstrap.cc:492] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4: No bootstrap required, opened a new log
I20260810 22:28:35.621480 8955 ts_tablet_manager.cc:1397] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4: Time spent bootstrapping tablet: real 0.008s user 0.007s sys 0.000s
I20260810 22:28:35.621570 8953 ts_tablet_manager.cc:1397] T 1f99182d93ed483e9f520bf312a591e3 P 747982f4b834445b93f077dd1295b2ff: Time spent bootstrapping tablet: real 0.007s user 0.006s sys 0.000s
I20260810 22:28:35.623919 8954 raft_consensus.cc:348] T 459e1d55c65e4857a7ccba6b13316243 P d06d8f442c824758ae944ffe71e3cb44 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "747982f4b834445b93f077dd1295b2ff" member_type: VOTER last_known_addr { host: "127.8.18.129" port: 40911 } } peers { permanent_uuid: "01cff8a283e94658aea5fad9fa0b2db4" member_type: VOTER last_known_addr { host: "127.8.18.130" port: 33631 } } peers { permanent_uuid: "d06d8f442c824758ae944ffe71e3cb44" member_type: VOTER last_known_addr { host: "127.8.18.131" port: 36507 } }
I20260810 22:28:35.624254 8954 raft_consensus.cc:374] T 459e1d55c65e4857a7ccba6b13316243 P d06d8f442c824758ae944ffe71e3cb44 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20260810 22:28:35.624351 8954 raft_consensus.cc:729] T 459e1d55c65e4857a7ccba6b13316243 P d06d8f442c824758ae944ffe71e3cb44 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d06d8f442c824758ae944ffe71e3cb44, State: Initialized, Role: FOLLOWER
I20260810 22:28:35.624913 8954 consensus_queue.cc:260] T 459e1d55c65e4857a7ccba6b13316243 P d06d8f442c824758ae944ffe71e3cb44 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "747982f4b834445b93f077dd1295b2ff" member_type: VOTER last_known_addr { host: "127.8.18.129" port: 40911 } } peers { permanent_uuid: "01cff8a283e94658aea5fad9fa0b2db4" member_type: VOTER last_known_addr { host: "127.8.18.130" port: 33631 } } peers { permanent_uuid: "d06d8f442c824758ae944ffe71e3cb44" member_type: VOTER last_known_addr { host: "127.8.18.131" port: 36507 } }
I20260810 22:28:35.625535 8953 raft_consensus.cc:348] T 1f99182d93ed483e9f520bf312a591e3 P 747982f4b834445b93f077dd1295b2ff [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d06d8f442c824758ae944ffe71e3cb44" member_type: VOTER last_known_addr { host: "127.8.18.131" port: 36507 } } peers { permanent_uuid: "01cff8a283e94658aea5fad9fa0b2db4" member_type: VOTER last_known_addr { host: "127.8.18.130" port: 33631 } } peers { permanent_uuid: "747982f4b834445b93f077dd1295b2ff" member_type: VOTER last_known_addr { host: "127.8.18.129" port: 40911 } }
I20260810 22:28:35.625887 8953 raft_consensus.cc:374] T 1f99182d93ed483e9f520bf312a591e3 P 747982f4b834445b93f077dd1295b2ff [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20260810 22:28:35.625947 8953 raft_consensus.cc:729] T 1f99182d93ed483e9f520bf312a591e3 P 747982f4b834445b93f077dd1295b2ff [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 747982f4b834445b93f077dd1295b2ff, State: Initialized, Role: FOLLOWER
I20260810 22:28:35.625993 8933 heartbeater.cc:503] Master 127.8.18.190:40323 was elected leader, sending a full tablet report...
I20260810 22:28:35.626281 8954 ts_tablet_manager.cc:1428] T 459e1d55c65e4857a7ccba6b13316243 P d06d8f442c824758ae944ffe71e3cb44: Time spent starting tablet: real 0.008s user 0.004s sys 0.004s
I20260810 22:28:35.626379 8955 raft_consensus.cc:348] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "747982f4b834445b93f077dd1295b2ff" member_type: VOTER last_known_addr { host: "127.8.18.129" port: 40911 } } peers { permanent_uuid: "01cff8a283e94658aea5fad9fa0b2db4" member_type: VOTER last_known_addr { host: "127.8.18.130" port: 33631 } } peers { permanent_uuid: "d06d8f442c824758ae944ffe71e3cb44" member_type: VOTER last_known_addr { host: "127.8.18.131" port: 36507 } }
I20260810 22:28:35.626508 8953 consensus_queue.cc:260] T 1f99182d93ed483e9f520bf312a591e3 P 747982f4b834445b93f077dd1295b2ff [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d06d8f442c824758ae944ffe71e3cb44" member_type: VOTER last_known_addr { host: "127.8.18.131" port: 36507 } } peers { permanent_uuid: "01cff8a283e94658aea5fad9fa0b2db4" member_type: VOTER last_known_addr { host: "127.8.18.130" port: 33631 } } peers { permanent_uuid: "747982f4b834445b93f077dd1295b2ff" member_type: VOTER last_known_addr { host: "127.8.18.129" port: 40911 } }
I20260810 22:28:35.626629 8954 tablet_bootstrap.cc:492] T 1f99182d93ed483e9f520bf312a591e3 P d06d8f442c824758ae944ffe71e3cb44: Bootstrap starting.
I20260810 22:28:35.626637 8955 raft_consensus.cc:374] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20260810 22:28:35.626689 8955 raft_consensus.cc:729] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 01cff8a283e94658aea5fad9fa0b2db4, State: Initialized, Role: FOLLOWER
I20260810 22:28:35.627169 8955 consensus_queue.cc:260] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "747982f4b834445b93f077dd1295b2ff" member_type: VOTER last_known_addr { host: "127.8.18.129" port: 40911 } } peers { permanent_uuid: "01cff8a283e94658aea5fad9fa0b2db4" member_type: VOTER last_known_addr { host: "127.8.18.130" port: 33631 } } peers { permanent_uuid: "d06d8f442c824758ae944ffe71e3cb44" member_type: VOTER last_known_addr { host: "127.8.18.131" port: 36507 } }
I20260810 22:28:35.628098 8953 ts_tablet_manager.cc:1428] T 1f99182d93ed483e9f520bf312a591e3 P 747982f4b834445b93f077dd1295b2ff: Time spent starting tablet: real 0.006s user 0.006s sys 0.000s
I20260810 22:28:35.628214 8954 tablet_bootstrap.cc:654] T 1f99182d93ed483e9f520bf312a591e3 P d06d8f442c824758ae944ffe71e3cb44: Neither blocks nor log segments found. Creating new log.
I20260810 22:28:35.628302 8955 ts_tablet_manager.cc:1428] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4: Time spent starting tablet: real 0.007s user 0.006s sys 0.000s
I20260810 22:28:35.628567 8579 heartbeater.cc:503] Master 127.8.18.190:40323 was elected leader, sending a full tablet report...
I20260810 22:28:35.628676 8756 heartbeater.cc:503] Master 127.8.18.190:40323 was elected leader, sending a full tablet report...
I20260810 22:28:35.629310 8955 tablet_bootstrap.cc:492] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4: Bootstrap starting.
I20260810 22:28:35.630897 8955 tablet_bootstrap.cc:654] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4: Neither blocks nor log segments found. Creating new log.
I20260810 22:28:35.631065 8953 tablet_bootstrap.cc:492] T 459e1d55c65e4857a7ccba6b13316243 P 747982f4b834445b93f077dd1295b2ff: Bootstrap starting.
I20260810 22:28:35.632546 8953 tablet_bootstrap.cc:654] T 459e1d55c65e4857a7ccba6b13316243 P 747982f4b834445b93f077dd1295b2ff: Neither blocks nor log segments found. Creating new log.
I20260810 22:28:35.633253 8954 tablet_bootstrap.cc:492] T 1f99182d93ed483e9f520bf312a591e3 P d06d8f442c824758ae944ffe71e3cb44: No bootstrap required, opened a new log
I20260810 22:28:35.633438 8954 ts_tablet_manager.cc:1397] T 1f99182d93ed483e9f520bf312a591e3 P d06d8f442c824758ae944ffe71e3cb44: Time spent bootstrapping tablet: real 0.007s user 0.003s sys 0.001s
I20260810 22:28:35.633957 8955 tablet_bootstrap.cc:492] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4: No bootstrap required, opened a new log
I20260810 22:28:35.634065 8955 ts_tablet_manager.cc:1397] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4: Time spent bootstrapping tablet: real 0.005s user 0.000s sys 0.003s
I20260810 22:28:35.634207 8954 raft_consensus.cc:348] T 1f99182d93ed483e9f520bf312a591e3 P d06d8f442c824758ae944ffe71e3cb44 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d06d8f442c824758ae944ffe71e3cb44" member_type: VOTER last_known_addr { host: "127.8.18.131" port: 36507 } } peers { permanent_uuid: "01cff8a283e94658aea5fad9fa0b2db4" member_type: VOTER last_known_addr { host: "127.8.18.130" port: 33631 } } peers { permanent_uuid: "747982f4b834445b93f077dd1295b2ff" member_type: VOTER last_known_addr { host: "127.8.18.129" port: 40911 } }
I20260810 22:28:35.634392 8954 raft_consensus.cc:374] T 1f99182d93ed483e9f520bf312a591e3 P d06d8f442c824758ae944ffe71e3cb44 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20260810 22:28:35.634446 8954 raft_consensus.cc:729] T 1f99182d93ed483e9f520bf312a591e3 P d06d8f442c824758ae944ffe71e3cb44 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d06d8f442c824758ae944ffe71e3cb44, State: Initialized, Role: FOLLOWER
I20260810 22:28:35.634585 8954 consensus_queue.cc:260] T 1f99182d93ed483e9f520bf312a591e3 P d06d8f442c824758ae944ffe71e3cb44 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d06d8f442c824758ae944ffe71e3cb44" member_type: VOTER last_known_addr { host: "127.8.18.131" port: 36507 } } peers { permanent_uuid: "01cff8a283e94658aea5fad9fa0b2db4" member_type: VOTER last_known_addr { host: "127.8.18.130" port: 33631 } } peers { permanent_uuid: "747982f4b834445b93f077dd1295b2ff" member_type: VOTER last_known_addr { host: "127.8.18.129" port: 40911 } }
I20260810 22:28:35.634897 8954 ts_tablet_manager.cc:1428] T 1f99182d93ed483e9f520bf312a591e3 P d06d8f442c824758ae944ffe71e3cb44: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20260810 22:28:35.634897 8955 raft_consensus.cc:348] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "747982f4b834445b93f077dd1295b2ff" member_type: VOTER last_known_addr { host: "127.8.18.129" port: 40911 } } peers { permanent_uuid: "01cff8a283e94658aea5fad9fa0b2db4" member_type: VOTER last_known_addr { host: "127.8.18.130" port: 33631 } } peers { permanent_uuid: "d06d8f442c824758ae944ffe71e3cb44" member_type: VOTER last_known_addr { host: "127.8.18.131" port: 36507 } }
I20260810 22:28:35.635022 8955 raft_consensus.cc:374] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20260810 22:28:35.635064 8955 raft_consensus.cc:729] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 01cff8a283e94658aea5fad9fa0b2db4, State: Initialized, Role: FOLLOWER
I20260810 22:28:35.635095 8954 tablet_bootstrap.cc:492] T b29f930721b943039d1df1f03e073a74 P d06d8f442c824758ae944ffe71e3cb44: Bootstrap starting.
I20260810 22:28:35.635202 8955 consensus_queue.cc:260] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "747982f4b834445b93f077dd1295b2ff" member_type: VOTER last_known_addr { host: "127.8.18.129" port: 40911 } } peers { permanent_uuid: "01cff8a283e94658aea5fad9fa0b2db4" member_type: VOTER last_known_addr { host: "127.8.18.130" port: 33631 } } peers { permanent_uuid: "d06d8f442c824758ae944ffe71e3cb44" member_type: VOTER last_known_addr { host: "127.8.18.131" port: 36507 } }
I20260810 22:28:35.635531 8955 ts_tablet_manager.cc:1428] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4: Time spent starting tablet: real 0.001s user 0.000s sys 0.002s
I20260810 22:28:35.635730 8955 tablet_bootstrap.cc:492] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4: Bootstrap starting.
I20260810 22:28:35.636677 8954 tablet_bootstrap.cc:654] T b29f930721b943039d1df1f03e073a74 P d06d8f442c824758ae944ffe71e3cb44: Neither blocks nor log segments found. Creating new log.
I20260810 22:28:35.636744 8953 tablet_bootstrap.cc:492] T 459e1d55c65e4857a7ccba6b13316243 P 747982f4b834445b93f077dd1295b2ff: No bootstrap required, opened a new log
I20260810 22:28:35.636831 8953 ts_tablet_manager.cc:1397] T 459e1d55c65e4857a7ccba6b13316243 P 747982f4b834445b93f077dd1295b2ff: Time spent bootstrapping tablet: real 0.006s user 0.001s sys 0.002s
I20260810 22:28:35.637209 8955 tablet_bootstrap.cc:654] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4: Neither blocks nor log segments found. Creating new log.
I20260810 22:28:35.637713 8953 raft_consensus.cc:348] T 459e1d55c65e4857a7ccba6b13316243 P 747982f4b834445b93f077dd1295b2ff [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "747982f4b834445b93f077dd1295b2ff" member_type: VOTER last_known_addr { host: "127.8.18.129" port: 40911 } } peers { permanent_uuid: "01cff8a283e94658aea5fad9fa0b2db4" member_type: VOTER last_known_addr { host: "127.8.18.130" port: 33631 } } peers { permanent_uuid: "d06d8f442c824758ae944ffe71e3cb44" member_type: VOTER last_known_addr { host: "127.8.18.131" port: 36507 } }
I20260810 22:28:35.637864 8953 raft_consensus.cc:374] T 459e1d55c65e4857a7ccba6b13316243 P 747982f4b834445b93f077dd1295b2ff [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20260810 22:28:35.637908 8953 raft_consensus.cc:729] T 459e1d55c65e4857a7ccba6b13316243 P 747982f4b834445b93f077dd1295b2ff [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 747982f4b834445b93f077dd1295b2ff, State: Initialized, Role: FOLLOWER
I20260810 22:28:35.638116 8953 consensus_queue.cc:260] T 459e1d55c65e4857a7ccba6b13316243 P 747982f4b834445b93f077dd1295b2ff [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "747982f4b834445b93f077dd1295b2ff" member_type: VOTER last_known_addr { host: "127.8.18.129" port: 40911 } } peers { permanent_uuid: "01cff8a283e94658aea5fad9fa0b2db4" member_type: VOTER last_known_addr { host: "127.8.18.130" port: 33631 } } peers { permanent_uuid: "d06d8f442c824758ae944ffe71e3cb44" member_type: VOTER last_known_addr { host: "127.8.18.131" port: 36507 } }
I20260810 22:28:35.638453 8953 ts_tablet_manager.cc:1428] T 459e1d55c65e4857a7ccba6b13316243 P 747982f4b834445b93f077dd1295b2ff: Time spent starting tablet: real 0.002s user 0.000s sys 0.001s
I20260810 22:28:35.638674 8953 tablet_bootstrap.cc:492] T b29f930721b943039d1df1f03e073a74 P 747982f4b834445b93f077dd1295b2ff: Bootstrap starting.
I20260810 22:28:35.639467 8954 tablet_bootstrap.cc:492] T b29f930721b943039d1df1f03e073a74 P d06d8f442c824758ae944ffe71e3cb44: No bootstrap required, opened a new log
I20260810 22:28:35.639572 8954 ts_tablet_manager.cc:1397] T b29f930721b943039d1df1f03e073a74 P d06d8f442c824758ae944ffe71e3cb44: Time spent bootstrapping tablet: real 0.005s user 0.001s sys 0.002s
I20260810 22:28:35.640152 8954 raft_consensus.cc:348] T b29f930721b943039d1df1f03e073a74 P d06d8f442c824758ae944ffe71e3cb44 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "747982f4b834445b93f077dd1295b2ff" member_type: VOTER last_known_addr { host: "127.8.18.129" port: 40911 } } peers { permanent_uuid: "01cff8a283e94658aea5fad9fa0b2db4" member_type: VOTER last_known_addr { host: "127.8.18.130" port: 33631 } } peers { permanent_uuid: "d06d8f442c824758ae944ffe71e3cb44" member_type: VOTER last_known_addr { host: "127.8.18.131" port: 36507 } }
I20260810 22:28:35.640311 8954 raft_consensus.cc:374] T b29f930721b943039d1df1f03e073a74 P d06d8f442c824758ae944ffe71e3cb44 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20260810 22:28:35.640350 8954 raft_consensus.cc:729] T b29f930721b943039d1df1f03e073a74 P d06d8f442c824758ae944ffe71e3cb44 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d06d8f442c824758ae944ffe71e3cb44, State: Initialized, Role: FOLLOWER
I20260810 22:28:35.640501 8954 consensus_queue.cc:260] T b29f930721b943039d1df1f03e073a74 P d06d8f442c824758ae944ffe71e3cb44 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "747982f4b834445b93f077dd1295b2ff" member_type: VOTER last_known_addr { host: "127.8.18.129" port: 40911 } } peers { permanent_uuid: "01cff8a283e94658aea5fad9fa0b2db4" member_type: VOTER last_known_addr { host: "127.8.18.130" port: 33631 } } peers { permanent_uuid: "d06d8f442c824758ae944ffe71e3cb44" member_type: VOTER last_known_addr { host: "127.8.18.131" port: 36507 } }
I20260810 22:28:35.640807 8954 ts_tablet_manager.cc:1428] T b29f930721b943039d1df1f03e073a74 P d06d8f442c824758ae944ffe71e3cb44: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20260810 22:28:35.641463 8955 tablet_bootstrap.cc:492] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4: No bootstrap required, opened a new log
I20260810 22:28:35.641481 8953 tablet_bootstrap.cc:654] T b29f930721b943039d1df1f03e073a74 P 747982f4b834445b93f077dd1295b2ff: Neither blocks nor log segments found. Creating new log.
I20260810 22:28:35.641573 8955 ts_tablet_manager.cc:1397] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4: Time spent bootstrapping tablet: real 0.006s user 0.003s sys 0.001s
I20260810 22:28:35.642904 8955 raft_consensus.cc:348] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d06d8f442c824758ae944ffe71e3cb44" member_type: VOTER last_known_addr { host: "127.8.18.131" port: 36507 } } peers { permanent_uuid: "01cff8a283e94658aea5fad9fa0b2db4" member_type: VOTER last_known_addr { host: "127.8.18.130" port: 33631 } } peers { permanent_uuid: "747982f4b834445b93f077dd1295b2ff" member_type: VOTER last_known_addr { host: "127.8.18.129" port: 40911 } }
I20260810 22:28:35.643103 8955 raft_consensus.cc:374] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20260810 22:28:35.643162 8955 raft_consensus.cc:729] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 01cff8a283e94658aea5fad9fa0b2db4, State: Initialized, Role: FOLLOWER
I20260810 22:28:35.643291 8955 consensus_queue.cc:260] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d06d8f442c824758ae944ffe71e3cb44" member_type: VOTER last_known_addr { host: "127.8.18.131" port: 36507 } } peers { permanent_uuid: "01cff8a283e94658aea5fad9fa0b2db4" member_type: VOTER last_known_addr { host: "127.8.18.130" port: 33631 } } peers { permanent_uuid: "747982f4b834445b93f077dd1295b2ff" member_type: VOTER last_known_addr { host: "127.8.18.129" port: 40911 } }
I20260810 22:28:35.643694 8953 tablet_bootstrap.cc:492] T b29f930721b943039d1df1f03e073a74 P 747982f4b834445b93f077dd1295b2ff: No bootstrap required, opened a new log
I20260810 22:28:35.643718 8955 ts_tablet_manager.cc:1428] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4: Time spent starting tablet: real 0.002s user 0.000s sys 0.001s
I20260810 22:28:35.643791 8953 ts_tablet_manager.cc:1397] T b29f930721b943039d1df1f03e073a74 P 747982f4b834445b93f077dd1295b2ff: Time spent bootstrapping tablet: real 0.005s user 0.002s sys 0.001s
I20260810 22:28:35.644486 8953 raft_consensus.cc:348] T b29f930721b943039d1df1f03e073a74 P 747982f4b834445b93f077dd1295b2ff [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "747982f4b834445b93f077dd1295b2ff" member_type: VOTER last_known_addr { host: "127.8.18.129" port: 40911 } } peers { permanent_uuid: "01cff8a283e94658aea5fad9fa0b2db4" member_type: VOTER last_known_addr { host: "127.8.18.130" port: 33631 } } peers { permanent_uuid: "d06d8f442c824758ae944ffe71e3cb44" member_type: VOTER last_known_addr { host: "127.8.18.131" port: 36507 } }
I20260810 22:28:35.644652 8953 raft_consensus.cc:374] T b29f930721b943039d1df1f03e073a74 P 747982f4b834445b93f077dd1295b2ff [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20260810 22:28:35.644708 8953 raft_consensus.cc:729] T b29f930721b943039d1df1f03e073a74 P 747982f4b834445b93f077dd1295b2ff [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 747982f4b834445b93f077dd1295b2ff, State: Initialized, Role: FOLLOWER
I20260810 22:28:35.644857 8953 consensus_queue.cc:260] T b29f930721b943039d1df1f03e073a74 P 747982f4b834445b93f077dd1295b2ff [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "747982f4b834445b93f077dd1295b2ff" member_type: VOTER last_known_addr { host: "127.8.18.129" port: 40911 } } peers { permanent_uuid: "01cff8a283e94658aea5fad9fa0b2db4" member_type: VOTER last_known_addr { host: "127.8.18.130" port: 33631 } } peers { permanent_uuid: "d06d8f442c824758ae944ffe71e3cb44" member_type: VOTER last_known_addr { host: "127.8.18.131" port: 36507 } }
I20260810 22:28:35.645169 8953 ts_tablet_manager.cc:1428] T b29f930721b943039d1df1f03e073a74 P 747982f4b834445b93f077dd1295b2ff: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20260810 22:28:35.645460 8961 raft_consensus.cc:482] T b29f930721b943039d1df1f03e073a74 P d06d8f442c824758ae944ffe71e3cb44 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20260810 22:28:35.645694 8961 raft_consensus.cc:504] T b29f930721b943039d1df1f03e073a74 P d06d8f442c824758ae944ffe71e3cb44 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "747982f4b834445b93f077dd1295b2ff" member_type: VOTER last_known_addr { host: "127.8.18.129" port: 40911 } } peers { permanent_uuid: "01cff8a283e94658aea5fad9fa0b2db4" member_type: VOTER last_known_addr { host: "127.8.18.130" port: 33631 } } peers { permanent_uuid: "d06d8f442c824758ae944ffe71e3cb44" member_type: VOTER last_known_addr { host: "127.8.18.131" port: 36507 } }
I20260810 22:28:35.647157 8961 leader_election.cc:288] T b29f930721b943039d1df1f03e073a74 P d06d8f442c824758ae944ffe71e3cb44 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911), 01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631)
I20260810 22:28:35.653264 8514 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b29f930721b943039d1df1f03e073a74" candidate_uuid: "d06d8f442c824758ae944ffe71e3cb44" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "747982f4b834445b93f077dd1295b2ff" is_pre_election: true
I20260810 22:28:35.653522 8696 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b29f930721b943039d1df1f03e073a74" candidate_uuid: "d06d8f442c824758ae944ffe71e3cb44" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "01cff8a283e94658aea5fad9fa0b2db4" is_pre_election: true
I20260810 22:28:35.653854 8514 raft_consensus.cc:2454] T b29f930721b943039d1df1f03e073a74 P 747982f4b834445b93f077dd1295b2ff [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate d06d8f442c824758ae944ffe71e3cb44 in term 0.
I20260810 22:28:35.654520 8780 leader_election.cc:302] T b29f930721b943039d1df1f03e073a74 P d06d8f442c824758ae944ffe71e3cb44 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 747982f4b834445b93f077dd1295b2ff, d06d8f442c824758ae944ffe71e3cb44; no voters:
I20260810 22:28:35.654824 8961 raft_consensus.cc:2781] T b29f930721b943039d1df1f03e073a74 P d06d8f442c824758ae944ffe71e3cb44 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20260810 22:28:35.654938 8961 raft_consensus.cc:482] T b29f930721b943039d1df1f03e073a74 P d06d8f442c824758ae944ffe71e3cb44 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20260810 22:28:35.654979 8961 raft_consensus.cc:3037] T b29f930721b943039d1df1f03e073a74 P d06d8f442c824758ae944ffe71e3cb44 [term 0 FOLLOWER]: Advancing to term 1
I20260810 22:28:35.655575 8696 raft_consensus.cc:2454] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate d06d8f442c824758ae944ffe71e3cb44 in term 0.
I20260810 22:28:35.656308 8961 raft_consensus.cc:504] T b29f930721b943039d1df1f03e073a74 P d06d8f442c824758ae944ffe71e3cb44 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "747982f4b834445b93f077dd1295b2ff" member_type: VOTER last_known_addr { host: "127.8.18.129" port: 40911 } } peers { permanent_uuid: "01cff8a283e94658aea5fad9fa0b2db4" member_type: VOTER last_known_addr { host: "127.8.18.130" port: 33631 } } peers { permanent_uuid: "d06d8f442c824758ae944ffe71e3cb44" member_type: VOTER last_known_addr { host: "127.8.18.131" port: 36507 } }
I20260810 22:28:35.656709 8961 leader_election.cc:288] T b29f930721b943039d1df1f03e073a74 P d06d8f442c824758ae944ffe71e3cb44 [CANDIDATE]: Term 1 election: Requested vote from peers 747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911), 01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631)
I20260810 22:28:35.657083 8514 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b29f930721b943039d1df1f03e073a74" candidate_uuid: "d06d8f442c824758ae944ffe71e3cb44" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "747982f4b834445b93f077dd1295b2ff"
I20260810 22:28:35.657179 8696 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b29f930721b943039d1df1f03e073a74" candidate_uuid: "d06d8f442c824758ae944ffe71e3cb44" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "01cff8a283e94658aea5fad9fa0b2db4"
I20260810 22:28:35.657230 8514 raft_consensus.cc:3037] T b29f930721b943039d1df1f03e073a74 P 747982f4b834445b93f077dd1295b2ff [term 0 FOLLOWER]: Advancing to term 1
I20260810 22:28:35.657392 8696 raft_consensus.cc:3037] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 [term 0 FOLLOWER]: Advancing to term 1
I20260810 22:28:35.658586 8514 raft_consensus.cc:2454] T b29f930721b943039d1df1f03e073a74 P 747982f4b834445b93f077dd1295b2ff [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate d06d8f442c824758ae944ffe71e3cb44 in term 1.
I20260810 22:28:35.658654 8696 raft_consensus.cc:2454] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate d06d8f442c824758ae944ffe71e3cb44 in term 1.
I20260810 22:28:35.658926 8780 leader_election.cc:302] T b29f930721b943039d1df1f03e073a74 P d06d8f442c824758ae944ffe71e3cb44 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 747982f4b834445b93f077dd1295b2ff, d06d8f442c824758ae944ffe71e3cb44; no voters:
I20260810 22:28:35.659178 8961 raft_consensus.cc:2781] T b29f930721b943039d1df1f03e073a74 P d06d8f442c824758ae944ffe71e3cb44 [term 1 FOLLOWER]: Leader election won for term 1
I20260810 22:28:35.659416 8961 raft_consensus.cc:686] T b29f930721b943039d1df1f03e073a74 P d06d8f442c824758ae944ffe71e3cb44 [term 1 LEADER]: Becoming Leader. State: Replica: d06d8f442c824758ae944ffe71e3cb44, State: Running, Role: LEADER
I20260810 22:28:35.659788 8961 consensus_queue.cc:237] T b29f930721b943039d1df1f03e073a74 P d06d8f442c824758ae944ffe71e3cb44 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "747982f4b834445b93f077dd1295b2ff" member_type: VOTER last_known_addr { host: "127.8.18.129" port: 40911 } } peers { permanent_uuid: "01cff8a283e94658aea5fad9fa0b2db4" member_type: VOTER last_known_addr { host: "127.8.18.130" port: 33631 } } peers { permanent_uuid: "d06d8f442c824758ae944ffe71e3cb44" member_type: VOTER last_known_addr { host: "127.8.18.131" port: 36507 } }
I20260810 22:28:35.660987 8963 raft_consensus.cc:482] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20260810 22:28:35.661140 8963 raft_consensus.cc:504] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "747982f4b834445b93f077dd1295b2ff" member_type: VOTER last_known_addr { host: "127.8.18.129" port: 40911 } } peers { permanent_uuid: "01cff8a283e94658aea5fad9fa0b2db4" member_type: VOTER last_known_addr { host: "127.8.18.130" port: 33631 } } peers { permanent_uuid: "d06d8f442c824758ae944ffe71e3cb44" member_type: VOTER last_known_addr { host: "127.8.18.131" port: 36507 } }
I20260810 22:28:35.662595 8963 leader_election.cc:288] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911), d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507)
I20260810 22:28:35.664323 8317 catalog_manager.cc:5495] T b29f930721b943039d1df1f03e073a74 P d06d8f442c824758ae944ffe71e3cb44 reported cstate change: term changed from 0 to 1, leader changed from <none> to d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131). New cstate: current_term: 1 leader_uuid: "d06d8f442c824758ae944ffe71e3cb44" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "747982f4b834445b93f077dd1295b2ff" member_type: VOTER last_known_addr { host: "127.8.18.129" port: 40911 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "01cff8a283e94658aea5fad9fa0b2db4" member_type: VOTER last_known_addr { host: "127.8.18.130" port: 33631 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "d06d8f442c824758ae944ffe71e3cb44" member_type: VOTER last_known_addr { host: "127.8.18.131" port: 36507 } health_report { overall_health: HEALTHY } } }
I20260810 22:28:35.667881 8514 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "459e1d55c65e4857a7ccba6b13316243" candidate_uuid: "01cff8a283e94658aea5fad9fa0b2db4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "747982f4b834445b93f077dd1295b2ff" is_pre_election: true
I20260810 22:28:35.668043 8514 raft_consensus.cc:2454] T 459e1d55c65e4857a7ccba6b13316243 P 747982f4b834445b93f077dd1295b2ff [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 01cff8a283e94658aea5fad9fa0b2db4 in term 0.
I20260810 22:28:35.668576 8608 leader_election.cc:302] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 01cff8a283e94658aea5fad9fa0b2db4, 747982f4b834445b93f077dd1295b2ff; no voters:
I20260810 22:28:35.668937 8963 raft_consensus.cc:2781] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20260810 22:28:35.669010 8963 raft_consensus.cc:482] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20260810 22:28:35.669057 8963 raft_consensus.cc:3037] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 [term 0 FOLLOWER]: Advancing to term 1
I20260810 22:28:35.669243 8869 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "459e1d55c65e4857a7ccba6b13316243" candidate_uuid: "01cff8a283e94658aea5fad9fa0b2db4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d06d8f442c824758ae944ffe71e3cb44" is_pre_election: true
I20260810 22:28:35.669504 8869 raft_consensus.cc:2454] T 459e1d55c65e4857a7ccba6b13316243 P d06d8f442c824758ae944ffe71e3cb44 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 01cff8a283e94658aea5fad9fa0b2db4 in term 0.
I20260810 22:28:35.670331 8963 raft_consensus.cc:504] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "747982f4b834445b93f077dd1295b2ff" member_type: VOTER last_known_addr { host: "127.8.18.129" port: 40911 } } peers { permanent_uuid: "01cff8a283e94658aea5fad9fa0b2db4" member_type: VOTER last_known_addr { host: "127.8.18.130" port: 33631 } } peers { permanent_uuid: "d06d8f442c824758ae944ffe71e3cb44" member_type: VOTER last_known_addr { host: "127.8.18.131" port: 36507 } }
I20260810 22:28:35.670776 8963 leader_election.cc:288] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 [CANDIDATE]: Term 1 election: Requested vote from peers 747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911), d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507)
I20260810 22:28:35.671166 8514 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "459e1d55c65e4857a7ccba6b13316243" candidate_uuid: "01cff8a283e94658aea5fad9fa0b2db4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "747982f4b834445b93f077dd1295b2ff"
I20260810 22:28:35.671324 8514 raft_consensus.cc:3037] T 459e1d55c65e4857a7ccba6b13316243 P 747982f4b834445b93f077dd1295b2ff [term 0 FOLLOWER]: Advancing to term 1
I20260810 22:28:35.671305 8869 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "459e1d55c65e4857a7ccba6b13316243" candidate_uuid: "01cff8a283e94658aea5fad9fa0b2db4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d06d8f442c824758ae944ffe71e3cb44"
I20260810 22:28:35.671428 8869 raft_consensus.cc:3037] T 459e1d55c65e4857a7ccba6b13316243 P d06d8f442c824758ae944ffe71e3cb44 [term 0 FOLLOWER]: Advancing to term 1
I20260810 22:28:35.672621 8514 raft_consensus.cc:2454] T 459e1d55c65e4857a7ccba6b13316243 P 747982f4b834445b93f077dd1295b2ff [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 01cff8a283e94658aea5fad9fa0b2db4 in term 1.
I20260810 22:28:35.672827 8869 raft_consensus.cc:2454] T 459e1d55c65e4857a7ccba6b13316243 P d06d8f442c824758ae944ffe71e3cb44 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 01cff8a283e94658aea5fad9fa0b2db4 in term 1.
I20260810 22:28:35.673274 8608 leader_election.cc:302] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 01cff8a283e94658aea5fad9fa0b2db4, 747982f4b834445b93f077dd1295b2ff; no voters:
I20260810 22:28:35.677537 8963 raft_consensus.cc:2781] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 [term 1 FOLLOWER]: Leader election won for term 1
I20260810 22:28:35.677865 8963 raft_consensus.cc:686] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 [term 1 LEADER]: Becoming Leader. State: Replica: 01cff8a283e94658aea5fad9fa0b2db4, State: Running, Role: LEADER
I20260810 22:28:35.678279 8963 consensus_queue.cc:237] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "747982f4b834445b93f077dd1295b2ff" member_type: VOTER last_known_addr { host: "127.8.18.129" port: 40911 } } peers { permanent_uuid: "01cff8a283e94658aea5fad9fa0b2db4" member_type: VOTER last_known_addr { host: "127.8.18.130" port: 33631 } } peers { permanent_uuid: "d06d8f442c824758ae944ffe71e3cb44" member_type: VOTER last_known_addr { host: "127.8.18.131" port: 36507 } }
I20260810 22:28:35.681861 8317 catalog_manager.cc:5495] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 reported cstate change: term changed from 0 to 1, leader changed from <none> to 01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130). New cstate: current_term: 1 leader_uuid: "01cff8a283e94658aea5fad9fa0b2db4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "747982f4b834445b93f077dd1295b2ff" member_type: VOTER last_known_addr { host: "127.8.18.129" port: 40911 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "01cff8a283e94658aea5fad9fa0b2db4" member_type: VOTER last_known_addr { host: "127.8.18.130" port: 33631 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d06d8f442c824758ae944ffe71e3cb44" member_type: VOTER last_known_addr { host: "127.8.18.131" port: 36507 } health_report { overall_health: UNKNOWN } } }
W20260810 22:28:35.784113 8934 tablet.cc:2369] T 1f99182d93ed483e9f520bf312a591e3 P d06d8f442c824758ae944ffe71e3cb44: Can't schedule compaction. Clean time has not been advanced past its initial value.
W20260810 22:28:35.788484 8580 tablet.cc:2369] T 1f99182d93ed483e9f520bf312a591e3 P 747982f4b834445b93f077dd1295b2ff: Can't schedule compaction. Clean time has not been advanced past its initial value.
W20260810 22:28:35.795080 8757 tablet.cc:2369] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4: Can't schedule compaction. Clean time has not been advanced past its initial value.
I20260810 22:28:35.997963 8963 raft_consensus.cc:482] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20260810 22:28:35.998139 8963 raft_consensus.cc:504] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d06d8f442c824758ae944ffe71e3cb44" member_type: VOTER last_known_addr { host: "127.8.18.131" port: 36507 } } peers { permanent_uuid: "01cff8a283e94658aea5fad9fa0b2db4" member_type: VOTER last_known_addr { host: "127.8.18.130" port: 33631 } } peers { permanent_uuid: "747982f4b834445b93f077dd1295b2ff" member_type: VOTER last_known_addr { host: "127.8.18.129" port: 40911 } }
I20260810 22:28:35.998699 8963 leader_election.cc:288] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507), 747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911)
I20260810 22:28:35.999224 8869 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1f99182d93ed483e9f520bf312a591e3" candidate_uuid: "01cff8a283e94658aea5fad9fa0b2db4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d06d8f442c824758ae944ffe71e3cb44" is_pre_election: true
I20260810 22:28:35.999281 8514 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1f99182d93ed483e9f520bf312a591e3" candidate_uuid: "01cff8a283e94658aea5fad9fa0b2db4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "747982f4b834445b93f077dd1295b2ff" is_pre_election: true
I20260810 22:28:35.999408 8869 raft_consensus.cc:2454] T 1f99182d93ed483e9f520bf312a591e3 P d06d8f442c824758ae944ffe71e3cb44 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 01cff8a283e94658aea5fad9fa0b2db4 in term 0.
I20260810 22:28:35.999420 8514 raft_consensus.cc:2454] T 1f99182d93ed483e9f520bf312a591e3 P 747982f4b834445b93f077dd1295b2ff [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 01cff8a283e94658aea5fad9fa0b2db4 in term 0.
I20260810 22:28:35.999811 8609 leader_election.cc:302] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 01cff8a283e94658aea5fad9fa0b2db4, d06d8f442c824758ae944ffe71e3cb44; no voters:
I20260810 22:28:36.000007 8963 raft_consensus.cc:2781] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20260810 22:28:36.000069 8963 raft_consensus.cc:482] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20260810 22:28:36.000128 8963 raft_consensus.cc:3037] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 [term 0 FOLLOWER]: Advancing to term 1
I20260810 22:28:36.001156 8963 raft_consensus.cc:504] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d06d8f442c824758ae944ffe71e3cb44" member_type: VOTER last_known_addr { host: "127.8.18.131" port: 36507 } } peers { permanent_uuid: "01cff8a283e94658aea5fad9fa0b2db4" member_type: VOTER last_known_addr { host: "127.8.18.130" port: 33631 } } peers { permanent_uuid: "747982f4b834445b93f077dd1295b2ff" member_type: VOTER last_known_addr { host: "127.8.18.129" port: 40911 } }
I20260810 22:28:36.001645 8963 leader_election.cc:288] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 [CANDIDATE]: Term 1 election: Requested vote from peers d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507), 747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911)
I20260810 22:28:36.002058 8869 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1f99182d93ed483e9f520bf312a591e3" candidate_uuid: "01cff8a283e94658aea5fad9fa0b2db4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d06d8f442c824758ae944ffe71e3cb44"
I20260810 22:28:36.002131 8514 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1f99182d93ed483e9f520bf312a591e3" candidate_uuid: "01cff8a283e94658aea5fad9fa0b2db4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "747982f4b834445b93f077dd1295b2ff"
I20260810 22:28:36.002207 8869 raft_consensus.cc:3037] T 1f99182d93ed483e9f520bf312a591e3 P d06d8f442c824758ae944ffe71e3cb44 [term 0 FOLLOWER]: Advancing to term 1
I20260810 22:28:36.002290 8514 raft_consensus.cc:3037] T 1f99182d93ed483e9f520bf312a591e3 P 747982f4b834445b93f077dd1295b2ff [term 0 FOLLOWER]: Advancing to term 1
I20260810 22:28:36.003191 8514 raft_consensus.cc:2454] T 1f99182d93ed483e9f520bf312a591e3 P 747982f4b834445b93f077dd1295b2ff [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 01cff8a283e94658aea5fad9fa0b2db4 in term 1.
I20260810 22:28:36.003201 8869 raft_consensus.cc:2454] T 1f99182d93ed483e9f520bf312a591e3 P d06d8f442c824758ae944ffe71e3cb44 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 01cff8a283e94658aea5fad9fa0b2db4 in term 1.
I20260810 22:28:36.003530 8608 leader_election.cc:302] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 01cff8a283e94658aea5fad9fa0b2db4, 747982f4b834445b93f077dd1295b2ff; no voters:
I20260810 22:28:36.003728 8963 raft_consensus.cc:2781] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 [term 1 FOLLOWER]: Leader election won for term 1
I20260810 22:28:36.003844 8963 raft_consensus.cc:686] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 [term 1 LEADER]: Becoming Leader. State: Replica: 01cff8a283e94658aea5fad9fa0b2db4, State: Running, Role: LEADER
I20260810 22:28:36.004004 8963 consensus_queue.cc:237] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d06d8f442c824758ae944ffe71e3cb44" member_type: VOTER last_known_addr { host: "127.8.18.131" port: 36507 } } peers { permanent_uuid: "01cff8a283e94658aea5fad9fa0b2db4" member_type: VOTER last_known_addr { host: "127.8.18.130" port: 33631 } } peers { permanent_uuid: "747982f4b834445b93f077dd1295b2ff" member_type: VOTER last_known_addr { host: "127.8.18.129" port: 40911 } }
I20260810 22:28:36.005957 8317 catalog_manager.cc:5495] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 reported cstate change: term changed from 0 to 1, leader changed from <none> to 01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130). New cstate: current_term: 1 leader_uuid: "01cff8a283e94658aea5fad9fa0b2db4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d06d8f442c824758ae944ffe71e3cb44" member_type: VOTER last_known_addr { host: "127.8.18.131" port: 36507 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "01cff8a283e94658aea5fad9fa0b2db4" member_type: VOTER last_known_addr { host: "127.8.18.130" port: 33631 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "747982f4b834445b93f077dd1295b2ff" member_type: VOTER last_known_addr { host: "127.8.18.129" port: 40911 } health_report { overall_health: UNKNOWN } } }
I20260810 22:28:36.066437 8963 consensus_queue.cc:1035] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d06d8f442c824758ae944ffe71e3cb44" member_type: VOTER last_known_addr { host: "127.8.18.131" port: 36507 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20260810 22:28:36.071537 8963 consensus_queue.cc:1035] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "747982f4b834445b93f077dd1295b2ff" member_type: VOTER last_known_addr { host: "127.8.18.129" port: 40911 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20260810 22:28:36.131340 8961 consensus_queue.cc:1035] T b29f930721b943039d1df1f03e073a74 P d06d8f442c824758ae944ffe71e3cb44 [LEADER]: Connected to new peer: Peer: permanent_uuid: "747982f4b834445b93f077dd1295b2ff" member_type: VOTER last_known_addr { host: "127.8.18.129" port: 40911 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20260810 22:28:36.136081 8961 consensus_queue.cc:1035] T b29f930721b943039d1df1f03e073a74 P d06d8f442c824758ae944ffe71e3cb44 [LEADER]: Connected to new peer: Peer: permanent_uuid: "01cff8a283e94658aea5fad9fa0b2db4" member_type: VOTER last_known_addr { host: "127.8.18.130" port: 33631 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20260810 22:28:36.430749 8963 consensus_queue.cc:1035] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d06d8f442c824758ae944ffe71e3cb44" member_type: VOTER last_known_addr { host: "127.8.18.131" port: 36507 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20260810 22:28:36.434653 8963 consensus_queue.cc:1035] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "747982f4b834445b93f077dd1295b2ff" member_type: VOTER last_known_addr { host: "127.8.18.129" port: 40911 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20260810 22:28:37.792562 8580 maintenance_manager.cc:419] P 747982f4b834445b93f077dd1295b2ff: Scheduling FlushMRSOp(1f99182d93ed483e9f520bf312a591e3): perf score=1.937278
I20260810 22:28:37.799122 8757 maintenance_manager.cc:419] P 01cff8a283e94658aea5fad9fa0b2db4: Scheduling FlushMRSOp(1f99182d93ed483e9f520bf312a591e3): perf score=1.937278
W20260810 22:28:37.812754 8937 outbound_call.cc:322] RPC callback for RPC call kudu.tserver.TabletServerService.Write -> {remote=127.8.18.130:33631, user_credentials={real_user=slave}} blocked reactor thread for 77348.9us
W20260810 22:28:38.001451 8937 outbound_call.cc:322] RPC callback for RPC call kudu.tserver.TabletServerService.Write -> {remote=127.8.18.130:33631, user_credentials={real_user=slave}} blocked reactor thread for 171920us
I20260810 22:28:38.039084 8934 maintenance_manager.cc:419] P d06d8f442c824758ae944ffe71e3cb44: Scheduling FlushMRSOp(1f99182d93ed483e9f520bf312a591e3): perf score=1.937278
W20260810 22:28:38.135136 8785 dir_manager.cc:174] Insufficient disk space under path /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-2/data/data: will retry after 10 seconds: IO error: Insufficient disk space to allocate 0 bytes under path /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-2/data/data (502484992 bytes available vs 518351011 bytes reserved) (error 28)
W20260810 22:28:38.157299 8785 tablet_replica_mm_ops.cc:214] T 1f99182d93ed483e9f520bf312a591e3 P d06d8f442c824758ae944ffe71e3cb44: failed to flush MRS: IO error: Failed to open DiskRowSet for flush: tablet 1f99182d93ed483e9f520bf312a591e3: unable to open output file for column varchar_val VARCHAR(65535) NULLABLE: No directories could be added (1 dirs total, 0 failed, 1 full): IO error: No directories available in 1f99182d93ed483e9f520bf312a591e3's directory group (1 dirs total, 0 failed, 1 full) (error 28) (error 19)
F20260810 22:28:38.157464 8785 tablet_replica_mm_ops.cc:215] Check failed: tablet->HasBeenStopped() Unrecoverable flush failure caused by error: IO error: Failed to open DiskRowSet for flush: tablet 1f99182d93ed483e9f520bf312a591e3: unable to open output file for column varchar_val VARCHAR(65535) NULLABLE: No directories could be added (1 dirs total, 0 failed, 1 full): IO error: No directories available in 1f99182d93ed483e9f520bf312a591e3's directory group (1 dirs total, 0 failed, 1 full) (error 28) (error 19)
*** Check failure stack trace: ***
@ 0x7fe031a899ad google::LogMessage::Fail() at ??:0
@ 0x7fe031a8c88a google::LogMessage::SendToLog() at ??:0
@ 0x7fe031a894de google::LogMessage::Flush() at ??:0
@ 0x7fe031a8aa19 google::LogMessageFatal::~LogMessageFatal() at ??:0
@ 0x7fe03a802299 kudu::tablet::FlushMRSOp::Perform() at ??:0
@ 0x7fe033c3f674 kudu::MaintenanceManager::LaunchOp() at ??:0
@ 0x7fe033c3c73c _ZZN4kudu18MaintenanceManager18RunSchedulerThreadEvENKUlvE_clEv at ??:0
@ 0x7fe033c4088c _ZNSt17_Function_handlerIFvvEZN4kudu18MaintenanceManager18RunSchedulerThreadEvEUlvE_E9_M_invokeERKSt9_Any_data at ??:0
@ 0x61d838 std::function<>::operator()() at ??:0
@ 0x7fe033cdb3d7 kudu::ThreadPool::DispatchThread() at ??:0
@ 0x7fe033cdbc83 _ZZN4kudu10ThreadPool12CreateThreadEvENKUlvE_clEv at ??:0
@ 0x7fe033cdd3f2 _ZNSt17_Function_handlerIFvvEZN4kudu10ThreadPool12CreateThreadEvEUlvE_E9_M_invokeERKSt9_Any_data at ??:0
@ 0x61d838 std::function<>::operator()() at ??:0
@ 0x7fe033ccc512 kudu::Thread::SuperviseThread() at ??:0
@ 0x7fe0353106db start_thread at ??:0
@ 0x7fe03058761f clone at ??:0
W20260810 22:28:38.660110 8938 meta_cache.cc:302] tablet b29f930721b943039d1df1f03e073a74: replica d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507) has failed: Network error: recv got EOF from 127.8.18.131:36507 (error 108)
W20260810 22:28:38.661401 8609 connection.cc:535] client connection to 127.8.18.131:36507 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
W20260810 22:28:38.661574 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
W20260810 22:28:38.669917 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
W20260810 22:28:38.670042 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20260810 22:28:38.879098 8670 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20260810 22:28:38.879122 8486 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20260810 22:28:38.912675 8426 maintenance_manager.cc:643] P 747982f4b834445b93f077dd1295b2ff: FlushMRSOp(1f99182d93ed483e9f520bf312a591e3) complete. Timing: real 1.119s user 0.626s sys 0.028s Metrics: {"bytes_written":436310,"cfile_init":1,"compiler_manager_pool.queue_time_us":54382,"compiler_manager_pool.run_cpu_time_us":300591,"compiler_manager_pool.run_wall_time_us":731865,"dirs.queue_time_us":196,"dirs.run_cpu_time_us":209,"dirs.run_wall_time_us":2885,"drs_written":1,"lbm_read_time_us":83,"lbm_reads_lt_1ms":4,"lbm_write_time_us":3572,"lbm_writes_lt_1ms":86,"peak_mem_usage":0,"rows_written":10000,"thread_start_us":333,"threads_started":3,"wal-append.queue_time_us":214}
I20260810 22:28:38.914211 8580 maintenance_manager.cc:419] P 747982f4b834445b93f077dd1295b2ff: Scheduling UndoDeltaBlockGCOp(1f99182d93ed483e9f520bf312a591e3): 62922 bytes on disk
I20260810 22:28:38.915077 8426 maintenance_manager.cc:643] P 747982f4b834445b93f077dd1295b2ff: UndoDeltaBlockGCOp(1f99182d93ed483e9f520bf312a591e3) complete. Timing: real 0.001s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":85,"lbm_reads_lt_1ms":4}
I20260810 22:28:38.915627 8580 maintenance_manager.cc:419] P 747982f4b834445b93f077dd1295b2ff: Scheduling FlushMRSOp(459e1d55c65e4857a7ccba6b13316243): perf score=1.937278
Master Summary
UUID | Address | Status
----------------------------------+--------------------+---------
e52d1e743df1434f88d46b1016be2ae7 | 127.8.18.190:40323 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.8.18.148:42749 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+--------------------+-------------+----------+----------------+-----------------
01cff8a283e94658aea5fad9fa0b2db4 | 127.8.18.130:33631 | HEALTHY | <none> | 2 | 0
747982f4b834445b93f077dd1295b2ff | 127.8.18.129:40911 | HEALTHY | <none> | 0 | 0
d06d8f442c824758ae944ffe71e3cb44 | 127.8.18.131:36507 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.8.18.131:36507: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+----------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.8.18.130:33631, 127.8.18.129:40911
ipki_server_key_size | 768 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.129 | experimental | 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.130 | experimental | 127.8.18.130:33631
never_fsync | true | unsafe,advanced | 127.8.18.130:33631, 127.8.18.129:40911
openssl_security_level_override | 0 | unsafe,hidden | 127.8.18.130:33631, 127.8.18.129:40911
rpc_server_allow_ephemeral_ports | true | unsafe | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_format | pb | hidden | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-0/data/info.pb | hidden | 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-1/data/info.pb | hidden | 127.8.18.130:33631
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+--------------------+----------------------------------------
builtin_ntp_servers | 127.8.18.148:42749 | 127.8.18.130:33631, 127.8.18.129:40911
time_source | builtin | 127.8.18.130:33631, 127.8.18.129:40911
Version Summary
Version | Servers
-----------------+-----------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.8.18.190:40323, tserver@127.8.18.130:33631, tserver@127.8.18.129:40911
Tablet Summary
Tablet 1f99182d93ed483e9f520bf312a591e3 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
All reported replicas are:
A = d06d8f442c824758ae944ffe71e3cb44
B = 01cff8a283e94658aea5fad9fa0b2db4
C = 747982f4b834445b93f077dd1295b2ff
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | -1 | Yes
C | A B* C | 1 | -1 | Yes
Tablet 459e1d55c65e4857a7ccba6b13316243 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet b29f930721b943039d1df1f03e073a74 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable [LEADER]
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | A B C* | 1 | -1 | Yes
B | A B C* | 1 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:28:38.924515 8266 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
I20260810 22:28:38.934785 8611 maintenance_manager.cc:643] P 01cff8a283e94658aea5fad9fa0b2db4: FlushMRSOp(1f99182d93ed483e9f520bf312a591e3) complete. Timing: real 1.135s user 0.618s sys 0.032s Metrics: {"bytes_written":436310,"cfile_init":1,"compiler_manager_pool.queue_time_us":24766,"compiler_manager_pool.run_cpu_time_us":298727,"compiler_manager_pool.run_wall_time_us":755764,"dirs.queue_time_us":644,"dirs.run_cpu_time_us":232,"dirs.run_wall_time_us":861,"drs_written":1,"lbm_read_time_us":69,"lbm_reads_lt_1ms":4,"lbm_write_time_us":7379,"lbm_writes_1-10_ms":1,"lbm_writes_lt_1ms":85,"peak_mem_usage":0,"rows_written":10000,"thread_start_us":264,"threads_started":3,"wal-append.queue_time_us":197}
I20260810 22:28:38.936252 8757 maintenance_manager.cc:419] P 01cff8a283e94658aea5fad9fa0b2db4: Scheduling UndoDeltaBlockGCOp(1f99182d93ed483e9f520bf312a591e3): 62922 bytes on disk
I20260810 22:28:38.936903 8611 maintenance_manager.cc:643] P 01cff8a283e94658aea5fad9fa0b2db4: UndoDeltaBlockGCOp(1f99182d93ed483e9f520bf312a591e3) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":60,"lbm_reads_lt_1ms":4}
I20260810 22:28:38.937412 8757 maintenance_manager.cc:419] P 01cff8a283e94658aea5fad9fa0b2db4: Scheduling FlushMRSOp(459e1d55c65e4857a7ccba6b13316243): perf score=1.937278
I20260810 22:28:39.163470 8670 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20260810 22:28:39.163758 8486 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+--------------------+---------
e52d1e743df1434f88d46b1016be2ae7 | 127.8.18.190:40323 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.8.18.148:42749 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+--------------------+-------------+----------+----------------+-----------------
01cff8a283e94658aea5fad9fa0b2db4 | 127.8.18.130:33631 | HEALTHY | <none> | 2 | 0
747982f4b834445b93f077dd1295b2ff | 127.8.18.129:40911 | HEALTHY | <none> | 0 | 0
d06d8f442c824758ae944ffe71e3cb44 | 127.8.18.131:36507 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.8.18.131:36507: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+----------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.8.18.130:33631, 127.8.18.129:40911
ipki_server_key_size | 768 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.129 | experimental | 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.130 | experimental | 127.8.18.130:33631
never_fsync | true | unsafe,advanced | 127.8.18.130:33631, 127.8.18.129:40911
openssl_security_level_override | 0 | unsafe,hidden | 127.8.18.130:33631, 127.8.18.129:40911
rpc_server_allow_ephemeral_ports | true | unsafe | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_format | pb | hidden | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-0/data/info.pb | hidden | 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-1/data/info.pb | hidden | 127.8.18.130:33631
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+--------------------+----------------------------------------
builtin_ntp_servers | 127.8.18.148:42749 | 127.8.18.130:33631, 127.8.18.129:40911
time_source | builtin | 127.8.18.130:33631, 127.8.18.129:40911
Version Summary
Version | Servers
-----------------+-----------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.8.18.190:40323, tserver@127.8.18.129:40911, tserver@127.8.18.130:33631
Tablet Summary
Tablet 1f99182d93ed483e9f520bf312a591e3 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
All reported replicas are:
A = d06d8f442c824758ae944ffe71e3cb44
B = 01cff8a283e94658aea5fad9fa0b2db4
C = 747982f4b834445b93f077dd1295b2ff
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | -1 | Yes
C | A B* C | 1 | -1 | Yes
Tablet 459e1d55c65e4857a7ccba6b13316243 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet b29f930721b943039d1df1f03e073a74 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable [LEADER]
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | A B C* | 1 | -1 | Yes
B | A B C* | 1 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:28:39.193920 8266 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
I20260810 22:28:39.340240 8426 maintenance_manager.cc:643] P 747982f4b834445b93f077dd1295b2ff: FlushMRSOp(459e1d55c65e4857a7ccba6b13316243) complete. Timing: real 0.424s user 0.416s sys 0.000s Metrics: {"bytes_written":471972,"cfile_init":1,"dirs.queue_time_us":82,"dirs.run_cpu_time_us":210,"dirs.run_wall_time_us":1625,"drs_written":1,"lbm_read_time_us":56,"lbm_reads_lt_1ms":4,"lbm_write_time_us":3025,"lbm_writes_lt_1ms":86,"peak_mem_usage":0,"rows_written":10000}
I20260810 22:28:39.340987 8580 maintenance_manager.cc:419] P 747982f4b834445b93f077dd1295b2ff: Scheduling UndoDeltaBlockGCOp(459e1d55c65e4857a7ccba6b13316243): 62922 bytes on disk
I20260810 22:28:39.341639 8426 maintenance_manager.cc:643] P 747982f4b834445b93f077dd1295b2ff: UndoDeltaBlockGCOp(459e1d55c65e4857a7ccba6b13316243) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":168,"lbm_reads_lt_1ms":4}
I20260810 22:28:39.342139 8580 maintenance_manager.cc:419] P 747982f4b834445b93f077dd1295b2ff: Scheduling FlushMRSOp(b29f930721b943039d1df1f03e073a74): perf score=1.937278
I20260810 22:28:39.350996 8611 maintenance_manager.cc:643] P 01cff8a283e94658aea5fad9fa0b2db4: FlushMRSOp(459e1d55c65e4857a7ccba6b13316243) complete. Timing: real 0.413s user 0.407s sys 0.004s Metrics: {"bytes_written":471972,"cfile_init":1,"dirs.queue_time_us":58,"dirs.run_cpu_time_us":229,"dirs.run_wall_time_us":1220,"drs_written":1,"lbm_read_time_us":44,"lbm_reads_lt_1ms":4,"lbm_write_time_us":2377,"lbm_writes_lt_1ms":86,"peak_mem_usage":0,"rows_written":10000,"thread_start_us":78,"threads_started":1,"wal-append.queue_time_us":209}
I20260810 22:28:39.351511 8757 maintenance_manager.cc:419] P 01cff8a283e94658aea5fad9fa0b2db4: Scheduling FlushMRSOp(b29f930721b943039d1df1f03e073a74): perf score=1.937278
I20260810 22:28:39.487380 8670 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20260810 22:28:39.489300 8486 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+--------------------+---------
e52d1e743df1434f88d46b1016be2ae7 | 127.8.18.190:40323 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.8.18.148:42749 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+--------------------+-------------+----------+----------------+-----------------
01cff8a283e94658aea5fad9fa0b2db4 | 127.8.18.130:33631 | HEALTHY | <none> | 2 | 0
747982f4b834445b93f077dd1295b2ff | 127.8.18.129:40911 | HEALTHY | <none> | 0 | 0
d06d8f442c824758ae944ffe71e3cb44 | 127.8.18.131:36507 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.8.18.131:36507: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+----------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.8.18.130:33631, 127.8.18.129:40911
ipki_server_key_size | 768 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.129 | experimental | 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.130 | experimental | 127.8.18.130:33631
never_fsync | true | unsafe,advanced | 127.8.18.130:33631, 127.8.18.129:40911
openssl_security_level_override | 0 | unsafe,hidden | 127.8.18.130:33631, 127.8.18.129:40911
rpc_server_allow_ephemeral_ports | true | unsafe | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_format | pb | hidden | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-0/data/info.pb | hidden | 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-1/data/info.pb | hidden | 127.8.18.130:33631
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+--------------------+----------------------------------------
builtin_ntp_servers | 127.8.18.148:42749 | 127.8.18.130:33631, 127.8.18.129:40911
time_source | builtin | 127.8.18.130:33631, 127.8.18.129:40911
Version Summary
Version | Servers
-----------------+-----------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.8.18.190:40323, tserver@127.8.18.130:33631, tserver@127.8.18.129:40911
Tablet Summary
Tablet 1f99182d93ed483e9f520bf312a591e3 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
All reported replicas are:
A = d06d8f442c824758ae944ffe71e3cb44
B = 01cff8a283e94658aea5fad9fa0b2db4
C = 747982f4b834445b93f077dd1295b2ff
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | -1 | Yes
C | A B* C | 1 | -1 | Yes
Tablet 459e1d55c65e4857a7ccba6b13316243 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet b29f930721b943039d1df1f03e073a74 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable [LEADER]
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | A B C* | 1 | -1 | Yes
B | A B C* | 1 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:28:39.518515 8266 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
I20260810 22:28:39.697388 9180 raft_consensus.cc:482] T b29f930721b943039d1df1f03e073a74 P 747982f4b834445b93f077dd1295b2ff [term 1 FOLLOWER]: Starting pre-election (detected failure of leader d06d8f442c824758ae944ffe71e3cb44)
I20260810 22:28:39.697574 9180 raft_consensus.cc:504] T b29f930721b943039d1df1f03e073a74 P 747982f4b834445b93f077dd1295b2ff [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "747982f4b834445b93f077dd1295b2ff" member_type: VOTER last_known_addr { host: "127.8.18.129" port: 40911 } } peers { permanent_uuid: "01cff8a283e94658aea5fad9fa0b2db4" member_type: VOTER last_known_addr { host: "127.8.18.130" port: 33631 } } peers { permanent_uuid: "d06d8f442c824758ae944ffe71e3cb44" member_type: VOTER last_known_addr { host: "127.8.18.131" port: 36507 } }
I20260810 22:28:39.699421 9180 leader_election.cc:288] T b29f930721b943039d1df1f03e073a74 P 747982f4b834445b93f077dd1295b2ff [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631), d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507)
W20260810 22:28:39.700017 8422 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111)
W20260810 22:28:39.701318 8422 leader_election.cc:334] T b29f930721b943039d1df1f03e073a74 P 747982f4b834445b93f077dd1295b2ff [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111)
I20260810 22:28:39.704764 8696 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b29f930721b943039d1df1f03e073a74" candidate_uuid: "747982f4b834445b93f077dd1295b2ff" candidate_term: 2 candidate_status { last_received { term: 1 index: 3 } } ignore_live_leader: false dest_uuid: "01cff8a283e94658aea5fad9fa0b2db4" is_pre_election: true
I20260810 22:28:39.705371 8421 leader_election.cc:302] T b29f930721b943039d1df1f03e073a74 P 747982f4b834445b93f077dd1295b2ff [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 747982f4b834445b93f077dd1295b2ff; no voters: 01cff8a283e94658aea5fad9fa0b2db4, d06d8f442c824758ae944ffe71e3cb44
I20260810 22:28:39.705669 9180 raft_consensus.cc:2726] T b29f930721b943039d1df1f03e073a74 P 747982f4b834445b93f077dd1295b2ff [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
I20260810 22:28:39.710597 8611 maintenance_manager.cc:643] P 01cff8a283e94658aea5fad9fa0b2db4: FlushMRSOp(b29f930721b943039d1df1f03e073a74) complete. Timing: real 0.359s user 0.352s sys 0.000s Metrics: {"bytes_written":472920,"cfile_init":1,"dirs.queue_time_us":63,"dirs.run_cpu_time_us":246,"dirs.run_wall_time_us":1395,"drs_written":1,"lbm_read_time_us":46,"lbm_reads_lt_1ms":4,"lbm_write_time_us":2255,"lbm_writes_lt_1ms":86,"peak_mem_usage":0,"rows_written":10000}
I20260810 22:28:39.711314 8757 maintenance_manager.cc:419] P 01cff8a283e94658aea5fad9fa0b2db4: Scheduling UndoDeltaBlockGCOp(b29f930721b943039d1df1f03e073a74): 63419 bytes on disk
I20260810 22:28:39.711364 9109 raft_consensus.cc:482] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader d06d8f442c824758ae944ffe71e3cb44)
I20260810 22:28:39.711503 9109 raft_consensus.cc:504] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "747982f4b834445b93f077dd1295b2ff" member_type: VOTER last_known_addr { host: "127.8.18.129" port: 40911 } } peers { permanent_uuid: "01cff8a283e94658aea5fad9fa0b2db4" member_type: VOTER last_known_addr { host: "127.8.18.130" port: 33631 } } peers { permanent_uuid: "d06d8f442c824758ae944ffe71e3cb44" member_type: VOTER last_known_addr { host: "127.8.18.131" port: 36507 } }
I20260810 22:28:39.711757 8611 maintenance_manager.cc:643] P 01cff8a283e94658aea5fad9fa0b2db4: UndoDeltaBlockGCOp(b29f930721b943039d1df1f03e073a74) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":77,"lbm_reads_lt_1ms":4}
I20260810 22:28:39.711984 9109 leader_election.cc:288] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911), d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507)
I20260810 22:28:39.712181 8757 maintenance_manager.cc:419] P 01cff8a283e94658aea5fad9fa0b2db4: Scheduling UndoDeltaBlockGCOp(459e1d55c65e4857a7ccba6b13316243): 62922 bytes on disk
I20260810 22:28:39.712479 8512 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b29f930721b943039d1df1f03e073a74" candidate_uuid: "01cff8a283e94658aea5fad9fa0b2db4" candidate_term: 2 candidate_status { last_received { term: 1 index: 3 } } ignore_live_leader: false dest_uuid: "747982f4b834445b93f077dd1295b2ff" is_pre_election: true
I20260810 22:28:39.712494 8611 maintenance_manager.cc:643] P 01cff8a283e94658aea5fad9fa0b2db4: UndoDeltaBlockGCOp(459e1d55c65e4857a7ccba6b13316243) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":60,"lbm_reads_lt_1ms":4}
I20260810 22:28:39.712630 8512 raft_consensus.cc:2454] T b29f930721b943039d1df1f03e073a74 P 747982f4b834445b93f077dd1295b2ff [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 01cff8a283e94658aea5fad9fa0b2db4 in term 1.
I20260810 22:28:39.712981 8608 leader_election.cc:302] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 01cff8a283e94658aea5fad9fa0b2db4, 747982f4b834445b93f077dd1295b2ff; no voters:
I20260810 22:28:39.713174 9109 raft_consensus.cc:2781] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 [term 1 FOLLOWER]: Leader pre-election won for term 2
W20260810 22:28:39.713176 8609 leader_election.cc:334] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111)
I20260810 22:28:39.713263 9109 raft_consensus.cc:482] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 [term 1 FOLLOWER]: Starting leader election (detected failure of leader d06d8f442c824758ae944ffe71e3cb44)
I20260810 22:28:39.713305 9109 raft_consensus.cc:3037] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 [term 1 FOLLOWER]: Advancing to term 2
I20260810 22:28:39.714484 9109 raft_consensus.cc:504] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "747982f4b834445b93f077dd1295b2ff" member_type: VOTER last_known_addr { host: "127.8.18.129" port: 40911 } } peers { permanent_uuid: "01cff8a283e94658aea5fad9fa0b2db4" member_type: VOTER last_known_addr { host: "127.8.18.130" port: 33631 } } peers { permanent_uuid: "d06d8f442c824758ae944ffe71e3cb44" member_type: VOTER last_known_addr { host: "127.8.18.131" port: 36507 } }
I20260810 22:28:39.714836 9109 leader_election.cc:288] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 [CANDIDATE]: Term 2 election: Requested vote from peers 747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911), d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507)
I20260810 22:28:39.715250 8512 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b29f930721b943039d1df1f03e073a74" candidate_uuid: "01cff8a283e94658aea5fad9fa0b2db4" candidate_term: 2 candidate_status { last_received { term: 1 index: 3 } } ignore_live_leader: false dest_uuid: "747982f4b834445b93f077dd1295b2ff"
I20260810 22:28:39.715401 8512 raft_consensus.cc:3037] T b29f930721b943039d1df1f03e073a74 P 747982f4b834445b93f077dd1295b2ff [term 1 FOLLOWER]: Advancing to term 2
W20260810 22:28:39.715971 8609 leader_election.cc:334] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111)
I20260810 22:28:39.716575 8512 raft_consensus.cc:2454] T b29f930721b943039d1df1f03e073a74 P 747982f4b834445b93f077dd1295b2ff [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 01cff8a283e94658aea5fad9fa0b2db4 in term 2.
I20260810 22:28:39.716897 8608 leader_election.cc:302] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 01cff8a283e94658aea5fad9fa0b2db4, 747982f4b834445b93f077dd1295b2ff; no voters: d06d8f442c824758ae944ffe71e3cb44
I20260810 22:28:39.717136 9109 raft_consensus.cc:2781] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 [term 2 FOLLOWER]: Leader election won for term 2
I20260810 22:28:39.717316 9109 raft_consensus.cc:686] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 [term 2 LEADER]: Becoming Leader. State: Replica: 01cff8a283e94658aea5fad9fa0b2db4, State: Running, Role: LEADER
I20260810 22:28:39.717500 9109 consensus_queue.cc:237] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 3, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "747982f4b834445b93f077dd1295b2ff" member_type: VOTER last_known_addr { host: "127.8.18.129" port: 40911 } } peers { permanent_uuid: "01cff8a283e94658aea5fad9fa0b2db4" member_type: VOTER last_known_addr { host: "127.8.18.130" port: 33631 } } peers { permanent_uuid: "d06d8f442c824758ae944ffe71e3cb44" member_type: VOTER last_known_addr { host: "127.8.18.131" port: 36507 } }
I20260810 22:28:39.719347 8312 catalog_manager.cc:5495] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 reported cstate change: term changed from 1 to 2, leader changed from d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131) to 01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130). New cstate: current_term: 2 leader_uuid: "01cff8a283e94658aea5fad9fa0b2db4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "747982f4b834445b93f077dd1295b2ff" member_type: VOTER last_known_addr { host: "127.8.18.129" port: 40911 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "01cff8a283e94658aea5fad9fa0b2db4" member_type: VOTER last_known_addr { host: "127.8.18.130" port: 33631 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d06d8f442c824758ae944ffe71e3cb44" member_type: VOTER last_known_addr { host: "127.8.18.131" port: 36507 } health_report { overall_health: UNKNOWN } } }
I20260810 22:28:39.720566 8426 maintenance_manager.cc:643] P 747982f4b834445b93f077dd1295b2ff: FlushMRSOp(b29f930721b943039d1df1f03e073a74) complete. Timing: real 0.378s user 0.375s sys 0.000s Metrics: {"bytes_written":472920,"cfile_init":1,"dirs.queue_time_us":91,"dirs.run_cpu_time_us":216,"dirs.run_wall_time_us":1038,"drs_written":1,"lbm_read_time_us":59,"lbm_reads_lt_1ms":4,"lbm_write_time_us":2375,"lbm_writes_lt_1ms":86,"peak_mem_usage":0,"rows_written":10000}
I20260810 22:28:39.721141 8580 maintenance_manager.cc:419] P 747982f4b834445b93f077dd1295b2ff: Scheduling UndoDeltaBlockGCOp(b29f930721b943039d1df1f03e073a74): 63419 bytes on disk
I20260810 22:28:39.721644 8426 maintenance_manager.cc:643] P 747982f4b834445b93f077dd1295b2ff: UndoDeltaBlockGCOp(b29f930721b943039d1df1f03e073a74) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":112,"lbm_reads_lt_1ms":4}
I20260810 22:28:39.911307 8670 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20260810 22:28:39.912405 8486 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+--------------------+---------
e52d1e743df1434f88d46b1016be2ae7 | 127.8.18.190:40323 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.8.18.148:42749 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+--------------------+-------------+----------+----------------+-----------------
01cff8a283e94658aea5fad9fa0b2db4 | 127.8.18.130:33631 | HEALTHY | <none> | 3 | 0
747982f4b834445b93f077dd1295b2ff | 127.8.18.129:40911 | HEALTHY | <none> | 0 | 0
d06d8f442c824758ae944ffe71e3cb44 | 127.8.18.131:36507 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.8.18.131:36507: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+----------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.8.18.130:33631, 127.8.18.129:40911
ipki_server_key_size | 768 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.129 | experimental | 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.130 | experimental | 127.8.18.130:33631
never_fsync | true | unsafe,advanced | 127.8.18.130:33631, 127.8.18.129:40911
openssl_security_level_override | 0 | unsafe,hidden | 127.8.18.130:33631, 127.8.18.129:40911
rpc_server_allow_ephemeral_ports | true | unsafe | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_format | pb | hidden | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-0/data/info.pb | hidden | 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-1/data/info.pb | hidden | 127.8.18.130:33631
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+--------------------+----------------------------------------
builtin_ntp_servers | 127.8.18.148:42749 | 127.8.18.130:33631, 127.8.18.129:40911
time_source | builtin | 127.8.18.130:33631, 127.8.18.129:40911
Version Summary
Version | Servers
-----------------+-----------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.8.18.190:40323, tserver@127.8.18.130:33631, tserver@127.8.18.129:40911
Tablet Summary
Tablet 1f99182d93ed483e9f520bf312a591e3 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
All reported replicas are:
A = d06d8f442c824758ae944ffe71e3cb44
B = 01cff8a283e94658aea5fad9fa0b2db4
C = 747982f4b834445b93f077dd1295b2ff
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | -1 | Yes
C | A B* C | 1 | -1 | Yes
Tablet 459e1d55c65e4857a7ccba6b13316243 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet b29f930721b943039d1df1f03e073a74 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B C | 2 | -1 | Yes
B | A B* C | 2 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:28:39.933511 8266 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:28:40.213212 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20260810 22:28:40.253760 8512 raft_consensus.cc:1261] T b29f930721b943039d1df1f03e073a74 P 747982f4b834445b93f077dd1295b2ff [term 2 FOLLOWER]: Refusing update from remote peer 01cff8a283e94658aea5fad9fa0b2db4: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 2 index: 4. (index mismatch)
I20260810 22:28:40.254354 9109 consensus_queue.cc:1035] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "747982f4b834445b93f077dd1295b2ff" member_type: VOTER last_known_addr { host: "127.8.18.129" port: 40911 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20260810 22:28:40.493105 8670 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20260810 22:28:40.493301 8486 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+--------------------+---------
e52d1e743df1434f88d46b1016be2ae7 | 127.8.18.190:40323 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.8.18.148:42749 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+--------------------+-------------+----------+----------------+-----------------
01cff8a283e94658aea5fad9fa0b2db4 | 127.8.18.130:33631 | HEALTHY | <none> | 3 | 0
747982f4b834445b93f077dd1295b2ff | 127.8.18.129:40911 | HEALTHY | <none> | 0 | 0
d06d8f442c824758ae944ffe71e3cb44 | 127.8.18.131:36507 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.8.18.131:36507: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+----------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.8.18.130:33631, 127.8.18.129:40911
ipki_server_key_size | 768 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.129 | experimental | 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.130 | experimental | 127.8.18.130:33631
never_fsync | true | unsafe,advanced | 127.8.18.130:33631, 127.8.18.129:40911
openssl_security_level_override | 0 | unsafe,hidden | 127.8.18.130:33631, 127.8.18.129:40911
rpc_server_allow_ephemeral_ports | true | unsafe | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_format | pb | hidden | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-0/data/info.pb | hidden | 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-1/data/info.pb | hidden | 127.8.18.130:33631
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+--------------------+----------------------------------------
builtin_ntp_servers | 127.8.18.148:42749 | 127.8.18.130:33631, 127.8.18.129:40911
time_source | builtin | 127.8.18.130:33631, 127.8.18.129:40911
Version Summary
Version | Servers
-----------------+-----------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.8.18.190:40323, tserver@127.8.18.130:33631, tserver@127.8.18.129:40911
Tablet Summary
Tablet 1f99182d93ed483e9f520bf312a591e3 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
All reported replicas are:
A = d06d8f442c824758ae944ffe71e3cb44
B = 01cff8a283e94658aea5fad9fa0b2db4
C = 747982f4b834445b93f077dd1295b2ff
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | -1 | Yes
C | A B* C | 1 | -1 | Yes
Tablet 459e1d55c65e4857a7ccba6b13316243 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet b29f930721b943039d1df1f03e073a74 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | -1 | Yes
B | A B* C | 2 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:28:40.518560 8266 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:28:40.841226 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
W20260810 22:28:40.845963 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
I20260810 22:28:41.332764 8670 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20260810 22:28:41.334410 8486 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+--------------------+---------
e52d1e743df1434f88d46b1016be2ae7 | 127.8.18.190:40323 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.8.18.148:42749 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+--------------------+-------------+----------+----------------+-----------------
01cff8a283e94658aea5fad9fa0b2db4 | 127.8.18.130:33631 | HEALTHY | <none> | 3 | 0
747982f4b834445b93f077dd1295b2ff | 127.8.18.129:40911 | HEALTHY | <none> | 0 | 0
d06d8f442c824758ae944ffe71e3cb44 | 127.8.18.131:36507 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.8.18.131:36507: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+----------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.8.18.130:33631, 127.8.18.129:40911
ipki_server_key_size | 768 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.129 | experimental | 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.130 | experimental | 127.8.18.130:33631
never_fsync | true | unsafe,advanced | 127.8.18.130:33631, 127.8.18.129:40911
openssl_security_level_override | 0 | unsafe,hidden | 127.8.18.130:33631, 127.8.18.129:40911
rpc_server_allow_ephemeral_ports | true | unsafe | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_format | pb | hidden | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-0/data/info.pb | hidden | 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-1/data/info.pb | hidden | 127.8.18.130:33631
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+--------------------+----------------------------------------
builtin_ntp_servers | 127.8.18.148:42749 | 127.8.18.130:33631, 127.8.18.129:40911
time_source | builtin | 127.8.18.130:33631, 127.8.18.129:40911
Version Summary
Version | Servers
-----------------+-----------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.8.18.190:40323, tserver@127.8.18.130:33631, tserver@127.8.18.129:40911
Tablet Summary
Tablet 1f99182d93ed483e9f520bf312a591e3 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
All reported replicas are:
A = d06d8f442c824758ae944ffe71e3cb44
B = 01cff8a283e94658aea5fad9fa0b2db4
C = 747982f4b834445b93f077dd1295b2ff
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | -1 | Yes
C | A B* C | 1 | -1 | Yes
Tablet 459e1d55c65e4857a7ccba6b13316243 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet b29f930721b943039d1df1f03e073a74 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | -1 | Yes
B | A B* C | 2 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:28:41.357563 8266 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:28:42.384931 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
I20260810 22:28:42.412744 8486 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20260810 22:28:42.413437 8670 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+--------------------+---------
e52d1e743df1434f88d46b1016be2ae7 | 127.8.18.190:40323 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.8.18.148:42749 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+--------------------+-------------+----------+----------------+-----------------
01cff8a283e94658aea5fad9fa0b2db4 | 127.8.18.130:33631 | HEALTHY | <none> | 3 | 0
747982f4b834445b93f077dd1295b2ff | 127.8.18.129:40911 | HEALTHY | <none> | 0 | 0
d06d8f442c824758ae944ffe71e3cb44 | 127.8.18.131:36507 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.8.18.131:36507: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+----------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.8.18.130:33631, 127.8.18.129:40911
ipki_server_key_size | 768 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.129 | experimental | 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.130 | experimental | 127.8.18.130:33631
never_fsync | true | unsafe,advanced | 127.8.18.130:33631, 127.8.18.129:40911
openssl_security_level_override | 0 | unsafe,hidden | 127.8.18.130:33631, 127.8.18.129:40911
rpc_server_allow_ephemeral_ports | true | unsafe | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_format | pb | hidden | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-0/data/info.pb | hidden | 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-1/data/info.pb | hidden | 127.8.18.130:33631
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+--------------------+----------------------------------------
builtin_ntp_servers | 127.8.18.148:42749 | 127.8.18.130:33631, 127.8.18.129:40911
time_source | builtin | 127.8.18.130:33631, 127.8.18.129:40911
Version Summary
Version | Servers
-----------------+-----------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.8.18.190:40323, tserver@127.8.18.129:40911, tserver@127.8.18.130:33631
Tablet Summary
Tablet 1f99182d93ed483e9f520bf312a591e3 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
All reported replicas are:
A = d06d8f442c824758ae944ffe71e3cb44
B = 01cff8a283e94658aea5fad9fa0b2db4
C = 747982f4b834445b93f077dd1295b2ff
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | -1 | Yes
C | A B* C | 1 | -1 | Yes
Tablet 459e1d55c65e4857a7ccba6b13316243 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet b29f930721b943039d1df1f03e073a74 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | -1 | Yes
B | A B* C | 2 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:28:42.438293 8266 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:28:42.919632 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
W20260810 22:28:43.240715 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
I20260810 22:28:43.492522 8486 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20260810 22:28:43.492585 8670 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+--------------------+---------
e52d1e743df1434f88d46b1016be2ae7 | 127.8.18.190:40323 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.8.18.148:42749 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+--------------------+-------------+----------+----------------+-----------------
01cff8a283e94658aea5fad9fa0b2db4 | 127.8.18.130:33631 | HEALTHY | <none> | 3 | 0
747982f4b834445b93f077dd1295b2ff | 127.8.18.129:40911 | HEALTHY | <none> | 0 | 0
d06d8f442c824758ae944ffe71e3cb44 | 127.8.18.131:36507 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.8.18.131:36507: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+----------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.8.18.130:33631, 127.8.18.129:40911
ipki_server_key_size | 768 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.129 | experimental | 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.130 | experimental | 127.8.18.130:33631
never_fsync | true | unsafe,advanced | 127.8.18.130:33631, 127.8.18.129:40911
openssl_security_level_override | 0 | unsafe,hidden | 127.8.18.130:33631, 127.8.18.129:40911
rpc_server_allow_ephemeral_ports | true | unsafe | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_format | pb | hidden | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-0/data/info.pb | hidden | 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-1/data/info.pb | hidden | 127.8.18.130:33631
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+--------------------+----------------------------------------
builtin_ntp_servers | 127.8.18.148:42749 | 127.8.18.130:33631, 127.8.18.129:40911
time_source | builtin | 127.8.18.130:33631, 127.8.18.129:40911
Version Summary
Version | Servers
-----------------+-----------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.8.18.190:40323, tserver@127.8.18.130:33631, tserver@127.8.18.129:40911
Tablet Summary
Tablet 1f99182d93ed483e9f520bf312a591e3 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
All reported replicas are:
A = d06d8f442c824758ae944ffe71e3cb44
B = 01cff8a283e94658aea5fad9fa0b2db4
C = 747982f4b834445b93f077dd1295b2ff
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | -1 | Yes
C | A B* C | 1 | -1 | Yes
Tablet 459e1d55c65e4857a7ccba6b13316243 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet b29f930721b943039d1df1f03e073a74 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | -1 | Yes
B | A B* C | 2 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:28:43.512593 8266 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:28:43.760347 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 32 similar messages]
I20260810 22:28:44.566783 8670 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20260810 22:28:44.567193 8486 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+--------------------+---------
e52d1e743df1434f88d46b1016be2ae7 | 127.8.18.190:40323 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.8.18.148:42749 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+--------------------+-------------+----------+----------------+-----------------
01cff8a283e94658aea5fad9fa0b2db4 | 127.8.18.130:33631 | HEALTHY | <none> | 3 | 0
747982f4b834445b93f077dd1295b2ff | 127.8.18.129:40911 | HEALTHY | <none> | 0 | 0
d06d8f442c824758ae944ffe71e3cb44 | 127.8.18.131:36507 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.8.18.131:36507: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+----------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.8.18.130:33631, 127.8.18.129:40911
ipki_server_key_size | 768 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.129 | experimental | 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.130 | experimental | 127.8.18.130:33631
never_fsync | true | unsafe,advanced | 127.8.18.130:33631, 127.8.18.129:40911
openssl_security_level_override | 0 | unsafe,hidden | 127.8.18.130:33631, 127.8.18.129:40911
rpc_server_allow_ephemeral_ports | true | unsafe | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_format | pb | hidden | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-0/data/info.pb | hidden | 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-1/data/info.pb | hidden | 127.8.18.130:33631
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+--------------------+----------------------------------------
builtin_ntp_servers | 127.8.18.148:42749 | 127.8.18.130:33631, 127.8.18.129:40911
time_source | builtin | 127.8.18.130:33631, 127.8.18.129:40911
Version Summary
Version | Servers
-----------------+-----------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.8.18.190:40323, tserver@127.8.18.130:33631, tserver@127.8.18.129:40911
Tablet Summary
Tablet 1f99182d93ed483e9f520bf312a591e3 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
All reported replicas are:
A = d06d8f442c824758ae944ffe71e3cb44
B = 01cff8a283e94658aea5fad9fa0b2db4
C = 747982f4b834445b93f077dd1295b2ff
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | -1 | Yes
C | A B* C | 1 | -1 | Yes
Tablet 459e1d55c65e4857a7ccba6b13316243 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet b29f930721b943039d1df1f03e073a74 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | -1 | Yes
B | A B* C | 2 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:28:44.586931 8266 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:28:44.947654 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
W20260810 22:28:45.275882 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
I20260810 22:28:45.650760 8670 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20260810 22:28:45.650797 8486 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+--------------------+---------
e52d1e743df1434f88d46b1016be2ae7 | 127.8.18.190:40323 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.8.18.148:42749 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+--------------------+-------------+----------+----------------+-----------------
01cff8a283e94658aea5fad9fa0b2db4 | 127.8.18.130:33631 | HEALTHY | <none> | 3 | 0
747982f4b834445b93f077dd1295b2ff | 127.8.18.129:40911 | HEALTHY | <none> | 0 | 0
d06d8f442c824758ae944ffe71e3cb44 | 127.8.18.131:36507 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.8.18.131:36507: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+----------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.8.18.130:33631, 127.8.18.129:40911
ipki_server_key_size | 768 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.129 | experimental | 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.130 | experimental | 127.8.18.130:33631
never_fsync | true | unsafe,advanced | 127.8.18.130:33631, 127.8.18.129:40911
openssl_security_level_override | 0 | unsafe,hidden | 127.8.18.130:33631, 127.8.18.129:40911
rpc_server_allow_ephemeral_ports | true | unsafe | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_format | pb | hidden | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-0/data/info.pb | hidden | 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-1/data/info.pb | hidden | 127.8.18.130:33631
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+--------------------+----------------------------------------
builtin_ntp_servers | 127.8.18.148:42749 | 127.8.18.130:33631, 127.8.18.129:40911
time_source | builtin | 127.8.18.130:33631, 127.8.18.129:40911
Version Summary
Version | Servers
-----------------+-----------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.8.18.190:40323, tserver@127.8.18.129:40911, tserver@127.8.18.130:33631
Tablet Summary
Tablet 1f99182d93ed483e9f520bf312a591e3 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
All reported replicas are:
A = d06d8f442c824758ae944ffe71e3cb44
B = 01cff8a283e94658aea5fad9fa0b2db4
C = 747982f4b834445b93f077dd1295b2ff
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | -1 | Yes
C | A B* C | 1 | -1 | Yes
Tablet 459e1d55c65e4857a7ccba6b13316243 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet b29f930721b943039d1df1f03e073a74 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | -1 | Yes
B | A B* C | 2 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:28:45.677031 8266 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:28:45.951637 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
I20260810 22:28:46.733585 8670 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20260810 22:28:46.734447 8486 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+--------------------+---------
e52d1e743df1434f88d46b1016be2ae7 | 127.8.18.190:40323 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.8.18.148:42749 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+--------------------+-------------+----------+----------------+-----------------
01cff8a283e94658aea5fad9fa0b2db4 | 127.8.18.130:33631 | HEALTHY | <none> | 3 | 0
747982f4b834445b93f077dd1295b2ff | 127.8.18.129:40911 | HEALTHY | <none> | 0 | 0
d06d8f442c824758ae944ffe71e3cb44 | 127.8.18.131:36507 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.8.18.131:36507: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+----------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.8.18.130:33631, 127.8.18.129:40911
ipki_server_key_size | 768 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.129 | experimental | 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.130 | experimental | 127.8.18.130:33631
never_fsync | true | unsafe,advanced | 127.8.18.130:33631, 127.8.18.129:40911
openssl_security_level_override | 0 | unsafe,hidden | 127.8.18.130:33631, 127.8.18.129:40911
rpc_server_allow_ephemeral_ports | true | unsafe | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_format | pb | hidden | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-0/data/info.pb | hidden | 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-1/data/info.pb | hidden | 127.8.18.130:33631
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+--------------------+----------------------------------------
builtin_ntp_servers | 127.8.18.148:42749 | 127.8.18.130:33631, 127.8.18.129:40911
time_source | builtin | 127.8.18.130:33631, 127.8.18.129:40911
Version Summary
Version | Servers
-----------------+-----------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.8.18.190:40323, tserver@127.8.18.130:33631, tserver@127.8.18.129:40911
Tablet Summary
Tablet 1f99182d93ed483e9f520bf312a591e3 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
All reported replicas are:
A = d06d8f442c824758ae944ffe71e3cb44
B = 01cff8a283e94658aea5fad9fa0b2db4
C = 747982f4b834445b93f077dd1295b2ff
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | -1 | Yes
C | A B* C | 1 | -1 | Yes
Tablet 459e1d55c65e4857a7ccba6b13316243 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet b29f930721b943039d1df1f03e073a74 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | -1 | Yes
B | A B* C | 2 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:28:46.759479 8266 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:28:47.381529 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
W20260810 22:28:47.691428 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 21: this message will repeat every 5th retry.
I20260810 22:28:47.815869 8670 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20260810 22:28:47.815929 8486 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+--------------------+---------
e52d1e743df1434f88d46b1016be2ae7 | 127.8.18.190:40323 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.8.18.148:42749 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+--------------------+-------------+----------+----------------+-----------------
01cff8a283e94658aea5fad9fa0b2db4 | 127.8.18.130:33631 | HEALTHY | <none> | 3 | 0
747982f4b834445b93f077dd1295b2ff | 127.8.18.129:40911 | HEALTHY | <none> | 0 | 0
d06d8f442c824758ae944ffe71e3cb44 | 127.8.18.131:36507 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.8.18.131:36507: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+----------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.8.18.130:33631, 127.8.18.129:40911
ipki_server_key_size | 768 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.129 | experimental | 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.130 | experimental | 127.8.18.130:33631
never_fsync | true | unsafe,advanced | 127.8.18.130:33631, 127.8.18.129:40911
openssl_security_level_override | 0 | unsafe,hidden | 127.8.18.130:33631, 127.8.18.129:40911
rpc_server_allow_ephemeral_ports | true | unsafe | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_format | pb | hidden | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-0/data/info.pb | hidden | 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-1/data/info.pb | hidden | 127.8.18.130:33631
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+--------------------+----------------------------------------
builtin_ntp_servers | 127.8.18.148:42749 | 127.8.18.130:33631, 127.8.18.129:40911
time_source | builtin | 127.8.18.130:33631, 127.8.18.129:40911
Version Summary
Version | Servers
-----------------+-----------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.8.18.190:40323, tserver@127.8.18.129:40911, tserver@127.8.18.130:33631
Tablet Summary
Tablet 1f99182d93ed483e9f520bf312a591e3 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
All reported replicas are:
A = d06d8f442c824758ae944ffe71e3cb44
B = 01cff8a283e94658aea5fad9fa0b2db4
C = 747982f4b834445b93f077dd1295b2ff
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | -1 | Yes
C | A B* C | 1 | -1 | Yes
Tablet 459e1d55c65e4857a7ccba6b13316243 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet b29f930721b943039d1df1f03e073a74 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | -1 | Yes
B | A B* C | 2 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:28:47.841383 8266 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:28:48.375842 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 21: this message will repeat every 5th retry.
W20260810 22:28:48.837621 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 30 similar messages]
I20260810 22:28:48.896639 8670 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20260810 22:28:48.897179 8486 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+--------------------+---------
e52d1e743df1434f88d46b1016be2ae7 | 127.8.18.190:40323 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.8.18.148:42749 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+--------------------+-------------+----------+----------------+-----------------
01cff8a283e94658aea5fad9fa0b2db4 | 127.8.18.130:33631 | HEALTHY | <none> | 3 | 0
747982f4b834445b93f077dd1295b2ff | 127.8.18.129:40911 | HEALTHY | <none> | 0 | 0
d06d8f442c824758ae944ffe71e3cb44 | 127.8.18.131:36507 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.8.18.131:36507: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+----------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.8.18.130:33631, 127.8.18.129:40911
ipki_server_key_size | 768 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.129 | experimental | 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.130 | experimental | 127.8.18.130:33631
never_fsync | true | unsafe,advanced | 127.8.18.130:33631, 127.8.18.129:40911
openssl_security_level_override | 0 | unsafe,hidden | 127.8.18.130:33631, 127.8.18.129:40911
rpc_server_allow_ephemeral_ports | true | unsafe | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_format | pb | hidden | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-0/data/info.pb | hidden | 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-1/data/info.pb | hidden | 127.8.18.130:33631
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+--------------------+----------------------------------------
builtin_ntp_servers | 127.8.18.148:42749 | 127.8.18.130:33631, 127.8.18.129:40911
time_source | builtin | 127.8.18.130:33631, 127.8.18.129:40911
Version Summary
Version | Servers
-----------------+-----------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.8.18.190:40323, tserver@127.8.18.130:33631, tserver@127.8.18.129:40911
Tablet Summary
Tablet 1f99182d93ed483e9f520bf312a591e3 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
All reported replicas are:
A = d06d8f442c824758ae944ffe71e3cb44
B = 01cff8a283e94658aea5fad9fa0b2db4
C = 747982f4b834445b93f077dd1295b2ff
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | -1 | Yes
C | A B* C | 1 | -1 | Yes
Tablet 459e1d55c65e4857a7ccba6b13316243 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet b29f930721b943039d1df1f03e073a74 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | -1 | Yes
B | A B* C | 2 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:28:48.921295 8266 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:28:49.828912 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 21: this message will repeat every 5th retry.
I20260810 22:28:49.978863 8670 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20260810 22:28:49.979094 8486 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+--------------------+---------
e52d1e743df1434f88d46b1016be2ae7 | 127.8.18.190:40323 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.8.18.148:42749 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+--------------------+-------------+----------+----------------+-----------------
01cff8a283e94658aea5fad9fa0b2db4 | 127.8.18.130:33631 | HEALTHY | <none> | 3 | 0
747982f4b834445b93f077dd1295b2ff | 127.8.18.129:40911 | HEALTHY | <none> | 0 | 0
d06d8f442c824758ae944ffe71e3cb44 | 127.8.18.131:36507 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.8.18.131:36507: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+----------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.8.18.130:33631, 127.8.18.129:40911
ipki_server_key_size | 768 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.129 | experimental | 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.130 | experimental | 127.8.18.130:33631
never_fsync | true | unsafe,advanced | 127.8.18.130:33631, 127.8.18.129:40911
openssl_security_level_override | 0 | unsafe,hidden | 127.8.18.130:33631, 127.8.18.129:40911
rpc_server_allow_ephemeral_ports | true | unsafe | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_format | pb | hidden | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-0/data/info.pb | hidden | 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-1/data/info.pb | hidden | 127.8.18.130:33631
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+--------------------+----------------------------------------
builtin_ntp_servers | 127.8.18.148:42749 | 127.8.18.130:33631, 127.8.18.129:40911
time_source | builtin | 127.8.18.130:33631, 127.8.18.129:40911
Version Summary
Version | Servers
-----------------+-----------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.8.18.190:40323, tserver@127.8.18.130:33631, tserver@127.8.18.129:40911
Tablet Summary
Tablet 1f99182d93ed483e9f520bf312a591e3 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
All reported replicas are:
A = d06d8f442c824758ae944ffe71e3cb44
B = 01cff8a283e94658aea5fad9fa0b2db4
C = 747982f4b834445b93f077dd1295b2ff
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | -1 | Yes
C | A B* C | 1 | -1 | Yes
Tablet 459e1d55c65e4857a7ccba6b13316243 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet b29f930721b943039d1df1f03e073a74 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | -1 | Yes
B | A B* C | 2 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:28:49.999102 8266 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:28:50.290094 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 26: this message will repeat every 5th retry.
I20260810 22:28:51.058175 8670 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20260810 22:28:51.058221 8486 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+--------------------+---------
e52d1e743df1434f88d46b1016be2ae7 | 127.8.18.190:40323 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.8.18.148:42749 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+--------------------+-------------+----------+----------------+-----------------
01cff8a283e94658aea5fad9fa0b2db4 | 127.8.18.130:33631 | HEALTHY | <none> | 3 | 0
747982f4b834445b93f077dd1295b2ff | 127.8.18.129:40911 | HEALTHY | <none> | 0 | 0
d06d8f442c824758ae944ffe71e3cb44 | 127.8.18.131:36507 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.8.18.131:36507: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+----------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.8.18.130:33631, 127.8.18.129:40911
ipki_server_key_size | 768 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.129 | experimental | 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.130 | experimental | 127.8.18.130:33631
never_fsync | true | unsafe,advanced | 127.8.18.130:33631, 127.8.18.129:40911
openssl_security_level_override | 0 | unsafe,hidden | 127.8.18.130:33631, 127.8.18.129:40911
rpc_server_allow_ephemeral_ports | true | unsafe | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_format | pb | hidden | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-0/data/info.pb | hidden | 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-1/data/info.pb | hidden | 127.8.18.130:33631
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+--------------------+----------------------------------------
builtin_ntp_servers | 127.8.18.148:42749 | 127.8.18.130:33631, 127.8.18.129:40911
time_source | builtin | 127.8.18.130:33631, 127.8.18.129:40911
Version Summary
Version | Servers
-----------------+-----------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.8.18.190:40323, tserver@127.8.18.130:33631, tserver@127.8.18.129:40911
Tablet Summary
Tablet 1f99182d93ed483e9f520bf312a591e3 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
All reported replicas are:
A = d06d8f442c824758ae944ffe71e3cb44
B = 01cff8a283e94658aea5fad9fa0b2db4
C = 747982f4b834445b93f077dd1295b2ff
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | -1 | Yes
C | A B* C | 1 | -1 | Yes
Tablet 459e1d55c65e4857a7ccba6b13316243 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet b29f930721b943039d1df1f03e073a74 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | -1 | Yes
B | A B* C | 2 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:28:51.080415 8266 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:28:51.177448 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 26: this message will repeat every 5th retry.
I20260810 22:28:52.153178 8670 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20260810 22:28:52.158172 8486 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+--------------------+---------
e52d1e743df1434f88d46b1016be2ae7 | 127.8.18.190:40323 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.8.18.148:42749 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+--------------------+-------------+----------+----------------+-----------------
01cff8a283e94658aea5fad9fa0b2db4 | 127.8.18.130:33631 | HEALTHY | <none> | 3 | 0
747982f4b834445b93f077dd1295b2ff | 127.8.18.129:40911 | HEALTHY | <none> | 0 | 0
d06d8f442c824758ae944ffe71e3cb44 | 127.8.18.131:36507 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.8.18.131:36507: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+----------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.8.18.130:33631, 127.8.18.129:40911
ipki_server_key_size | 768 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.129 | experimental | 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.130 | experimental | 127.8.18.130:33631
never_fsync | true | unsafe,advanced | 127.8.18.130:33631, 127.8.18.129:40911
openssl_security_level_override | 0 | unsafe,hidden | 127.8.18.130:33631, 127.8.18.129:40911
rpc_server_allow_ephemeral_ports | true | unsafe | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_format | pb | hidden | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-0/data/info.pb | hidden | 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-1/data/info.pb | hidden | 127.8.18.130:33631
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+--------------------+----------------------------------------
builtin_ntp_servers | 127.8.18.148:42749 | 127.8.18.130:33631, 127.8.18.129:40911
time_source | builtin | 127.8.18.130:33631, 127.8.18.129:40911
Version Summary
Version | Servers
-----------------+-----------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.8.18.190:40323, tserver@127.8.18.130:33631, tserver@127.8.18.129:40911
Tablet Summary
Tablet 1f99182d93ed483e9f520bf312a591e3 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
All reported replicas are:
A = d06d8f442c824758ae944ffe71e3cb44
B = 01cff8a283e94658aea5fad9fa0b2db4
C = 747982f4b834445b93f077dd1295b2ff
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | -1 | Yes
C | A B* C | 1 | -1 | Yes
Tablet 459e1d55c65e4857a7ccba6b13316243 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet b29f930721b943039d1df1f03e073a74 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | -1 | Yes
B | A B* C | 2 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:28:52.181551 8266 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:28:52.400949 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 26: this message will repeat every 5th retry.
W20260810 22:28:52.819913 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 31: this message will repeat every 5th retry.
I20260810 22:28:53.244163 8670 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20260810 22:28:53.244504 8486 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+--------------------+---------
e52d1e743df1434f88d46b1016be2ae7 | 127.8.18.190:40323 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.8.18.148:42749 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+--------------------+-------------+----------+----------------+-----------------
01cff8a283e94658aea5fad9fa0b2db4 | 127.8.18.130:33631 | HEALTHY | <none> | 3 | 0
747982f4b834445b93f077dd1295b2ff | 127.8.18.129:40911 | HEALTHY | <none> | 0 | 0
d06d8f442c824758ae944ffe71e3cb44 | 127.8.18.131:36507 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.8.18.131:36507: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+----------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.8.18.130:33631, 127.8.18.129:40911
ipki_server_key_size | 768 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.129 | experimental | 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.130 | experimental | 127.8.18.130:33631
never_fsync | true | unsafe,advanced | 127.8.18.130:33631, 127.8.18.129:40911
openssl_security_level_override | 0 | unsafe,hidden | 127.8.18.130:33631, 127.8.18.129:40911
rpc_server_allow_ephemeral_ports | true | unsafe | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_format | pb | hidden | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-0/data/info.pb | hidden | 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-1/data/info.pb | hidden | 127.8.18.130:33631
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+--------------------+----------------------------------------
builtin_ntp_servers | 127.8.18.148:42749 | 127.8.18.130:33631, 127.8.18.129:40911
time_source | builtin | 127.8.18.130:33631, 127.8.18.129:40911
Version Summary
Version | Servers
-----------------+-----------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.8.18.190:40323, tserver@127.8.18.130:33631, tserver@127.8.18.129:40911
Tablet Summary
Tablet 1f99182d93ed483e9f520bf312a591e3 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
All reported replicas are:
A = d06d8f442c824758ae944ffe71e3cb44
B = 01cff8a283e94658aea5fad9fa0b2db4
C = 747982f4b834445b93f077dd1295b2ff
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | -1 | Yes
C | A B* C | 1 | -1 | Yes
Tablet 459e1d55c65e4857a7ccba6b13316243 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet b29f930721b943039d1df1f03e073a74 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | -1 | Yes
B | A B* C | 2 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:28:53.266438 8266 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:28:53.803363 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 31: this message will repeat every 5th retry.
W20260810 22:28:53.955528 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 29 similar messages]
I20260810 22:28:54.324077 8670 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20260810 22:28:54.324419 8486 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+--------------------+---------
e52d1e743df1434f88d46b1016be2ae7 | 127.8.18.190:40323 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.8.18.148:42749 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+--------------------+-------------+----------+----------------+-----------------
01cff8a283e94658aea5fad9fa0b2db4 | 127.8.18.130:33631 | HEALTHY | <none> | 3 | 0
747982f4b834445b93f077dd1295b2ff | 127.8.18.129:40911 | HEALTHY | <none> | 0 | 0
d06d8f442c824758ae944ffe71e3cb44 | 127.8.18.131:36507 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.8.18.131:36507: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+----------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.8.18.130:33631, 127.8.18.129:40911
ipki_server_key_size | 768 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.129 | experimental | 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.130 | experimental | 127.8.18.130:33631
never_fsync | true | unsafe,advanced | 127.8.18.130:33631, 127.8.18.129:40911
openssl_security_level_override | 0 | unsafe,hidden | 127.8.18.130:33631, 127.8.18.129:40911
rpc_server_allow_ephemeral_ports | true | unsafe | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_format | pb | hidden | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-0/data/info.pb | hidden | 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-1/data/info.pb | hidden | 127.8.18.130:33631
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+--------------------+----------------------------------------
builtin_ntp_servers | 127.8.18.148:42749 | 127.8.18.130:33631, 127.8.18.129:40911
time_source | builtin | 127.8.18.130:33631, 127.8.18.129:40911
Version Summary
Version | Servers
-----------------+-----------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.8.18.190:40323, tserver@127.8.18.130:33631, tserver@127.8.18.129:40911
Tablet Summary
Tablet 1f99182d93ed483e9f520bf312a591e3 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
All reported replicas are:
A = d06d8f442c824758ae944ffe71e3cb44
B = 01cff8a283e94658aea5fad9fa0b2db4
C = 747982f4b834445b93f077dd1295b2ff
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | -1 | Yes
C | A B* C | 1 | -1 | Yes
Tablet 459e1d55c65e4857a7ccba6b13316243 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet b29f930721b943039d1df1f03e073a74 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | -1 | Yes
B | A B* C | 2 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:28:54.344238 8266 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:28:54.913241 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 31: this message will repeat every 5th retry.
I20260810 22:28:55.400540 8486 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20260810 22:28:55.400882 8670 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+--------------------+---------
e52d1e743df1434f88d46b1016be2ae7 | 127.8.18.190:40323 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.8.18.148:42749 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+--------------------+-------------+----------+----------------+-----------------
01cff8a283e94658aea5fad9fa0b2db4 | 127.8.18.130:33631 | HEALTHY | <none> | 3 | 0
747982f4b834445b93f077dd1295b2ff | 127.8.18.129:40911 | HEALTHY | <none> | 0 | 0
d06d8f442c824758ae944ffe71e3cb44 | 127.8.18.131:36507 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.8.18.131:36507: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+----------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.8.18.130:33631, 127.8.18.129:40911
ipki_server_key_size | 768 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.129 | experimental | 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.130 | experimental | 127.8.18.130:33631
never_fsync | true | unsafe,advanced | 127.8.18.130:33631, 127.8.18.129:40911
openssl_security_level_override | 0 | unsafe,hidden | 127.8.18.130:33631, 127.8.18.129:40911
rpc_server_allow_ephemeral_ports | true | unsafe | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_format | pb | hidden | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-0/data/info.pb | hidden | 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-1/data/info.pb | hidden | 127.8.18.130:33631
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+--------------------+----------------------------------------
builtin_ntp_servers | 127.8.18.148:42749 | 127.8.18.130:33631, 127.8.18.129:40911
time_source | builtin | 127.8.18.130:33631, 127.8.18.129:40911
Version Summary
Version | Servers
-----------------+-----------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.8.18.190:40323, tserver@127.8.18.129:40911, tserver@127.8.18.130:33631
Tablet Summary
Tablet 1f99182d93ed483e9f520bf312a591e3 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
All reported replicas are:
A = d06d8f442c824758ae944ffe71e3cb44
B = 01cff8a283e94658aea5fad9fa0b2db4
C = 747982f4b834445b93f077dd1295b2ff
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | -1 | Yes
C | A B* C | 1 | -1 | Yes
Tablet 459e1d55c65e4857a7ccba6b13316243 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet b29f930721b943039d1df1f03e073a74 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | -1 | Yes
B | A B* C | 2 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:28:55.424480 8266 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:28:55.487087 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 36: this message will repeat every 5th retry.
W20260810 22:28:56.246810 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 36: this message will repeat every 5th retry.
I20260810 22:28:56.480945 8670 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20260810 22:28:56.480980 8486 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+--------------------+---------
e52d1e743df1434f88d46b1016be2ae7 | 127.8.18.190:40323 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.8.18.148:42749 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+--------------------+-------------+----------+----------------+-----------------
01cff8a283e94658aea5fad9fa0b2db4 | 127.8.18.130:33631 | HEALTHY | <none> | 3 | 0
747982f4b834445b93f077dd1295b2ff | 127.8.18.129:40911 | HEALTHY | <none> | 0 | 0
d06d8f442c824758ae944ffe71e3cb44 | 127.8.18.131:36507 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.8.18.131:36507: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+----------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.8.18.130:33631, 127.8.18.129:40911
ipki_server_key_size | 768 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.129 | experimental | 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.130 | experimental | 127.8.18.130:33631
never_fsync | true | unsafe,advanced | 127.8.18.130:33631, 127.8.18.129:40911
openssl_security_level_override | 0 | unsafe,hidden | 127.8.18.130:33631, 127.8.18.129:40911
rpc_server_allow_ephemeral_ports | true | unsafe | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_format | pb | hidden | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-0/data/info.pb | hidden | 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-1/data/info.pb | hidden | 127.8.18.130:33631
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+--------------------+----------------------------------------
builtin_ntp_servers | 127.8.18.148:42749 | 127.8.18.130:33631, 127.8.18.129:40911
time_source | builtin | 127.8.18.130:33631, 127.8.18.129:40911
Version Summary
Version | Servers
-----------------+-----------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.8.18.190:40323, tserver@127.8.18.130:33631, tserver@127.8.18.129:40911
Tablet Summary
Tablet 1f99182d93ed483e9f520bf312a591e3 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
All reported replicas are:
A = d06d8f442c824758ae944ffe71e3cb44
B = 01cff8a283e94658aea5fad9fa0b2db4
C = 747982f4b834445b93f077dd1295b2ff
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | -1 | Yes
C | A B* C | 1 | -1 | Yes
Tablet 459e1d55c65e4857a7ccba6b13316243 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet b29f930721b943039d1df1f03e073a74 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | -1 | Yes
B | A B* C | 2 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:28:56.506371 8266 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:28:57.361851 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 36: this message will repeat every 5th retry.
I20260810 22:28:57.561270 8670 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20260810 22:28:57.561564 8486 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+--------------------+---------
e52d1e743df1434f88d46b1016be2ae7 | 127.8.18.190:40323 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.8.18.148:42749 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+--------------------+-------------+----------+----------------+-----------------
01cff8a283e94658aea5fad9fa0b2db4 | 127.8.18.130:33631 | HEALTHY | <none> | 3 | 0
747982f4b834445b93f077dd1295b2ff | 127.8.18.129:40911 | HEALTHY | <none> | 0 | 0
d06d8f442c824758ae944ffe71e3cb44 | 127.8.18.131:36507 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.8.18.131:36507: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+----------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.8.18.130:33631, 127.8.18.129:40911
ipki_server_key_size | 768 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.129 | experimental | 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.130 | experimental | 127.8.18.130:33631
never_fsync | true | unsafe,advanced | 127.8.18.130:33631, 127.8.18.129:40911
openssl_security_level_override | 0 | unsafe,hidden | 127.8.18.130:33631, 127.8.18.129:40911
rpc_server_allow_ephemeral_ports | true | unsafe | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_format | pb | hidden | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-0/data/info.pb | hidden | 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-1/data/info.pb | hidden | 127.8.18.130:33631
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+--------------------+----------------------------------------
builtin_ntp_servers | 127.8.18.148:42749 | 127.8.18.130:33631, 127.8.18.129:40911
time_source | builtin | 127.8.18.130:33631, 127.8.18.129:40911
Version Summary
Version | Servers
-----------------+-----------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.8.18.190:40323, tserver@127.8.18.130:33631, tserver@127.8.18.129:40911
Tablet Summary
Tablet 1f99182d93ed483e9f520bf312a591e3 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
All reported replicas are:
A = d06d8f442c824758ae944ffe71e3cb44
B = 01cff8a283e94658aea5fad9fa0b2db4
C = 747982f4b834445b93f077dd1295b2ff
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | -1 | Yes
C | A B* C | 1 | -1 | Yes
Tablet 459e1d55c65e4857a7ccba6b13316243 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet b29f930721b943039d1df1f03e073a74 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | -1 | Yes
B | A B* C | 2 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:28:57.581749 8266 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:28:57.760897 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 41: this message will repeat every 5th retry.
I20260810 22:28:58.636744 8670 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20260810 22:28:58.637042 8486 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+--------------------+---------
e52d1e743df1434f88d46b1016be2ae7 | 127.8.18.190:40323 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.8.18.148:42749 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+--------------------+-------------+----------+----------------+-----------------
01cff8a283e94658aea5fad9fa0b2db4 | 127.8.18.130:33631 | HEALTHY | <none> | 3 | 0
747982f4b834445b93f077dd1295b2ff | 127.8.18.129:40911 | HEALTHY | <none> | 0 | 0
d06d8f442c824758ae944ffe71e3cb44 | 127.8.18.131:36507 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.8.18.131:36507: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+----------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.8.18.130:33631, 127.8.18.129:40911
ipki_server_key_size | 768 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.129 | experimental | 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.130 | experimental | 127.8.18.130:33631
never_fsync | true | unsafe,advanced | 127.8.18.130:33631, 127.8.18.129:40911
openssl_security_level_override | 0 | unsafe,hidden | 127.8.18.130:33631, 127.8.18.129:40911
rpc_server_allow_ephemeral_ports | true | unsafe | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_format | pb | hidden | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-0/data/info.pb | hidden | 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-1/data/info.pb | hidden | 127.8.18.130:33631
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+--------------------+----------------------------------------
builtin_ntp_servers | 127.8.18.148:42749 | 127.8.18.130:33631, 127.8.18.129:40911
time_source | builtin | 127.8.18.130:33631, 127.8.18.129:40911
Version Summary
Version | Servers
-----------------+-----------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.8.18.190:40323, tserver@127.8.18.130:33631, tserver@127.8.18.129:40911
Tablet Summary
Tablet 1f99182d93ed483e9f520bf312a591e3 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
All reported replicas are:
A = d06d8f442c824758ae944ffe71e3cb44
B = 01cff8a283e94658aea5fad9fa0b2db4
C = 747982f4b834445b93f077dd1295b2ff
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | -1 | Yes
C | A B* C | 1 | -1 | Yes
Tablet 459e1d55c65e4857a7ccba6b13316243 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet b29f930721b943039d1df1f03e073a74 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | -1 | Yes
B | A B* C | 2 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:28:58.655900 8266 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:28:58.660856 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 41: this message will repeat every 5th retry.
W20260810 22:28:59.114264 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 30 similar messages]
I20260810 22:28:59.709519 8670 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20260810 22:28:59.709841 8486 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+--------------------+---------
e52d1e743df1434f88d46b1016be2ae7 | 127.8.18.190:40323 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.8.18.148:42749 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+--------------------+-------------+----------+----------------+-----------------
01cff8a283e94658aea5fad9fa0b2db4 | 127.8.18.130:33631 | HEALTHY | <none> | 3 | 0
747982f4b834445b93f077dd1295b2ff | 127.8.18.129:40911 | HEALTHY | <none> | 0 | 0
d06d8f442c824758ae944ffe71e3cb44 | 127.8.18.131:36507 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.8.18.131:36507: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+----------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.8.18.130:33631, 127.8.18.129:40911
ipki_server_key_size | 768 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.129 | experimental | 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.130 | experimental | 127.8.18.130:33631
never_fsync | true | unsafe,advanced | 127.8.18.130:33631, 127.8.18.129:40911
openssl_security_level_override | 0 | unsafe,hidden | 127.8.18.130:33631, 127.8.18.129:40911
rpc_server_allow_ephemeral_ports | true | unsafe | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_format | pb | hidden | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-0/data/info.pb | hidden | 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-1/data/info.pb | hidden | 127.8.18.130:33631
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+--------------------+----------------------------------------
builtin_ntp_servers | 127.8.18.148:42749 | 127.8.18.130:33631, 127.8.18.129:40911
time_source | builtin | 127.8.18.130:33631, 127.8.18.129:40911
Version Summary
Version | Servers
-----------------+-----------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.8.18.190:40323, tserver@127.8.18.130:33631, tserver@127.8.18.129:40911
Tablet Summary
Tablet 1f99182d93ed483e9f520bf312a591e3 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
All reported replicas are:
A = d06d8f442c824758ae944ffe71e3cb44
B = 01cff8a283e94658aea5fad9fa0b2db4
C = 747982f4b834445b93f077dd1295b2ff
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | -1 | Yes
C | A B* C | 1 | -1 | Yes
Tablet 459e1d55c65e4857a7ccba6b13316243 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet b29f930721b943039d1df1f03e073a74 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | -1 | Yes
B | A B* C | 2 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:28:59.728967 8266 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:28:59.753052 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 41: this message will repeat every 5th retry.
W20260810 22:29:00.296722 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 46: this message will repeat every 5th retry.
I20260810 22:29:00.815075 8670 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20260810 22:29:00.815075 8486 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+--------------------+---------
e52d1e743df1434f88d46b1016be2ae7 | 127.8.18.190:40323 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.8.18.148:42749 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+--------------------+-------------+----------+----------------+-----------------
01cff8a283e94658aea5fad9fa0b2db4 | 127.8.18.130:33631 | HEALTHY | <none> | 3 | 0
747982f4b834445b93f077dd1295b2ff | 127.8.18.129:40911 | HEALTHY | <none> | 0 | 0
d06d8f442c824758ae944ffe71e3cb44 | 127.8.18.131:36507 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.8.18.131:36507: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+----------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.8.18.130:33631, 127.8.18.129:40911
ipki_server_key_size | 768 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.129 | experimental | 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.130 | experimental | 127.8.18.130:33631
never_fsync | true | unsafe,advanced | 127.8.18.130:33631, 127.8.18.129:40911
openssl_security_level_override | 0 | unsafe,hidden | 127.8.18.130:33631, 127.8.18.129:40911
rpc_server_allow_ephemeral_ports | true | unsafe | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_format | pb | hidden | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-0/data/info.pb | hidden | 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-1/data/info.pb | hidden | 127.8.18.130:33631
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+--------------------+----------------------------------------
builtin_ntp_servers | 127.8.18.148:42749 | 127.8.18.130:33631, 127.8.18.129:40911
time_source | builtin | 127.8.18.130:33631, 127.8.18.129:40911
Version Summary
Version | Servers
-----------------+-----------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.8.18.190:40323, tserver@127.8.18.129:40911, tserver@127.8.18.130:33631
Tablet Summary
Tablet 1f99182d93ed483e9f520bf312a591e3 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
All reported replicas are:
A = d06d8f442c824758ae944ffe71e3cb44
B = 01cff8a283e94658aea5fad9fa0b2db4
C = 747982f4b834445b93f077dd1295b2ff
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | -1 | Yes
C | A B* C | 1 | -1 | Yes
Tablet 459e1d55c65e4857a7ccba6b13316243 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet b29f930721b943039d1df1f03e073a74 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | -1 | Yes
B | A B* C | 2 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:29:00.852890 8266 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:29:01.249699 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 46: this message will repeat every 5th retry.
I20260810 22:29:01.914552 8670 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20260810 22:29:01.914695 8486 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+--------------------+---------
e52d1e743df1434f88d46b1016be2ae7 | 127.8.18.190:40323 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.8.18.148:42749 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+--------------------+-------------+----------+----------------+-----------------
01cff8a283e94658aea5fad9fa0b2db4 | 127.8.18.130:33631 | HEALTHY | <none> | 3 | 0
747982f4b834445b93f077dd1295b2ff | 127.8.18.129:40911 | HEALTHY | <none> | 0 | 0
d06d8f442c824758ae944ffe71e3cb44 | 127.8.18.131:36507 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.8.18.131:36507: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+----------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.8.18.130:33631, 127.8.18.129:40911
ipki_server_key_size | 768 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.129 | experimental | 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.130 | experimental | 127.8.18.130:33631
never_fsync | true | unsafe,advanced | 127.8.18.130:33631, 127.8.18.129:40911
openssl_security_level_override | 0 | unsafe,hidden | 127.8.18.130:33631, 127.8.18.129:40911
rpc_server_allow_ephemeral_ports | true | unsafe | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_format | pb | hidden | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-0/data/info.pb | hidden | 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-1/data/info.pb | hidden | 127.8.18.130:33631
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+--------------------+----------------------------------------
builtin_ntp_servers | 127.8.18.148:42749 | 127.8.18.130:33631, 127.8.18.129:40911
time_source | builtin | 127.8.18.130:33631, 127.8.18.129:40911
Version Summary
Version | Servers
-----------------+-----------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.8.18.190:40323, tserver@127.8.18.130:33631, tserver@127.8.18.129:40911
Tablet Summary
Tablet 1f99182d93ed483e9f520bf312a591e3 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
All reported replicas are:
A = d06d8f442c824758ae944ffe71e3cb44
B = 01cff8a283e94658aea5fad9fa0b2db4
C = 747982f4b834445b93f077dd1295b2ff
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | -1 | Yes
C | A B* C | 1 | -1 | Yes
Tablet 459e1d55c65e4857a7ccba6b13316243 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet b29f930721b943039d1df1f03e073a74 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | -1 | Yes
B | A B* C | 2 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:29:01.933533 8266 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:29:02.287575 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 46: this message will repeat every 5th retry.
I20260810 22:29:02.985213 8670 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20260810 22:29:02.986395 8486 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+--------------------+---------
e52d1e743df1434f88d46b1016be2ae7 | 127.8.18.190:40323 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.8.18.148:42749 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+--------------------+-------------+----------+----------------+-----------------
01cff8a283e94658aea5fad9fa0b2db4 | 127.8.18.130:33631 | HEALTHY | <none> | 3 | 0
747982f4b834445b93f077dd1295b2ff | 127.8.18.129:40911 | HEALTHY | <none> | 0 | 0
d06d8f442c824758ae944ffe71e3cb44 | 127.8.18.131:36507 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.8.18.131:36507: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+----------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.8.18.130:33631, 127.8.18.129:40911
ipki_server_key_size | 768 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.129 | experimental | 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.130 | experimental | 127.8.18.130:33631
never_fsync | true | unsafe,advanced | 127.8.18.130:33631, 127.8.18.129:40911
openssl_security_level_override | 0 | unsafe,hidden | 127.8.18.130:33631, 127.8.18.129:40911
rpc_server_allow_ephemeral_ports | true | unsafe | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_format | pb | hidden | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-0/data/info.pb | hidden | 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-1/data/info.pb | hidden | 127.8.18.130:33631
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+--------------------+----------------------------------------
builtin_ntp_servers | 127.8.18.148:42749 | 127.8.18.130:33631, 127.8.18.129:40911
time_source | builtin | 127.8.18.130:33631, 127.8.18.129:40911
Version Summary
Version | Servers
-----------------+-----------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.8.18.190:40323, tserver@127.8.18.130:33631, tserver@127.8.18.129:40911
Tablet Summary
Tablet 1f99182d93ed483e9f520bf312a591e3 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
All reported replicas are:
A = d06d8f442c824758ae944ffe71e3cb44
B = 01cff8a283e94658aea5fad9fa0b2db4
C = 747982f4b834445b93f077dd1295b2ff
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | -1 | Yes
C | A B* C | 1 | -1 | Yes
Tablet 459e1d55c65e4857a7ccba6b13316243 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet b29f930721b943039d1df1f03e073a74 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | -1 | Yes
B | A B* C | 2 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:29:03.011382 8266 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:29:03.164613 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 51: this message will repeat every 5th retry.
W20260810 22:29:03.860119 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 51: this message will repeat every 5th retry.
I20260810 22:29:04.066407 8670 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20260810 22:29:04.066771 8486 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+--------------------+---------
e52d1e743df1434f88d46b1016be2ae7 | 127.8.18.190:40323 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.8.18.148:42749 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+--------------------+-------------+----------+----------------+-----------------
01cff8a283e94658aea5fad9fa0b2db4 | 127.8.18.130:33631 | HEALTHY | <none> | 3 | 0
747982f4b834445b93f077dd1295b2ff | 127.8.18.129:40911 | HEALTHY | <none> | 0 | 0
d06d8f442c824758ae944ffe71e3cb44 | 127.8.18.131:36507 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.8.18.131:36507: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+----------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.8.18.130:33631, 127.8.18.129:40911
ipki_server_key_size | 768 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.129 | experimental | 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.130 | experimental | 127.8.18.130:33631
never_fsync | true | unsafe,advanced | 127.8.18.130:33631, 127.8.18.129:40911
openssl_security_level_override | 0 | unsafe,hidden | 127.8.18.130:33631, 127.8.18.129:40911
rpc_server_allow_ephemeral_ports | true | unsafe | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_format | pb | hidden | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-0/data/info.pb | hidden | 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-1/data/info.pb | hidden | 127.8.18.130:33631
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+--------------------+----------------------------------------
builtin_ntp_servers | 127.8.18.148:42749 | 127.8.18.130:33631, 127.8.18.129:40911
time_source | builtin | 127.8.18.130:33631, 127.8.18.129:40911
Version Summary
Version | Servers
-----------------+-----------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.8.18.190:40323, tserver@127.8.18.130:33631, tserver@127.8.18.129:40911
Tablet Summary
Tablet 1f99182d93ed483e9f520bf312a591e3 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
All reported replicas are:
A = d06d8f442c824758ae944ffe71e3cb44
B = 01cff8a283e94658aea5fad9fa0b2db4
C = 747982f4b834445b93f077dd1295b2ff
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | -1 | Yes
C | A B* C | 1 | -1 | Yes
Tablet 459e1d55c65e4857a7ccba6b13316243 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet b29f930721b943039d1df1f03e073a74 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | -1 | Yes
B | A B* C | 2 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:29:04.088943 8266 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:29:04.178592 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 28 similar messages]
W20260810 22:29:04.744149 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 51: this message will repeat every 5th retry.
I20260810 22:29:05.144522 8670 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20260810 22:29:05.144606 8486 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+--------------------+---------
e52d1e743df1434f88d46b1016be2ae7 | 127.8.18.190:40323 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.8.18.148:42749 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+--------------------+-------------+----------+----------------+-----------------
01cff8a283e94658aea5fad9fa0b2db4 | 127.8.18.130:33631 | HEALTHY | <none> | 3 | 0
747982f4b834445b93f077dd1295b2ff | 127.8.18.129:40911 | HEALTHY | <none> | 0 | 0
d06d8f442c824758ae944ffe71e3cb44 | 127.8.18.131:36507 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.8.18.131:36507: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+----------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.8.18.130:33631, 127.8.18.129:40911
ipki_server_key_size | 768 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.129 | experimental | 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.130 | experimental | 127.8.18.130:33631
never_fsync | true | unsafe,advanced | 127.8.18.130:33631, 127.8.18.129:40911
openssl_security_level_override | 0 | unsafe,hidden | 127.8.18.130:33631, 127.8.18.129:40911
rpc_server_allow_ephemeral_ports | true | unsafe | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_format | pb | hidden | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-0/data/info.pb | hidden | 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-1/data/info.pb | hidden | 127.8.18.130:33631
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+--------------------+----------------------------------------
builtin_ntp_servers | 127.8.18.148:42749 | 127.8.18.130:33631, 127.8.18.129:40911
time_source | builtin | 127.8.18.130:33631, 127.8.18.129:40911
Version Summary
Version | Servers
-----------------+-----------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.8.18.190:40323, tserver@127.8.18.130:33631, tserver@127.8.18.129:40911
Tablet Summary
Tablet 1f99182d93ed483e9f520bf312a591e3 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
All reported replicas are:
A = d06d8f442c824758ae944ffe71e3cb44
B = 01cff8a283e94658aea5fad9fa0b2db4
C = 747982f4b834445b93f077dd1295b2ff
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | -1 | Yes
C | A B* C | 1 | -1 | Yes
Tablet 459e1d55c65e4857a7ccba6b13316243 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet b29f930721b943039d1df1f03e073a74 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | -1 | Yes
B | A B* C | 2 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:29:05.163230 8266 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:29:05.692757 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 56: this message will repeat every 5th retry.
W20260810 22:29:06.121992 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 56: this message will repeat every 5th retry.
I20260810 22:29:06.218497 8670 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20260810 22:29:06.218884 8486 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+--------------------+---------
e52d1e743df1434f88d46b1016be2ae7 | 127.8.18.190:40323 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.8.18.148:42749 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+--------------------+-------------+----------+----------------+-----------------
01cff8a283e94658aea5fad9fa0b2db4 | 127.8.18.130:33631 | HEALTHY | <none> | 3 | 0
747982f4b834445b93f077dd1295b2ff | 127.8.18.129:40911 | HEALTHY | <none> | 0 | 0
d06d8f442c824758ae944ffe71e3cb44 | 127.8.18.131:36507 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.8.18.131:36507: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+----------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.8.18.130:33631, 127.8.18.129:40911
ipki_server_key_size | 768 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.129 | experimental | 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.130 | experimental | 127.8.18.130:33631
never_fsync | true | unsafe,advanced | 127.8.18.130:33631, 127.8.18.129:40911
openssl_security_level_override | 0 | unsafe,hidden | 127.8.18.130:33631, 127.8.18.129:40911
rpc_server_allow_ephemeral_ports | true | unsafe | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_format | pb | hidden | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-0/data/info.pb | hidden | 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-1/data/info.pb | hidden | 127.8.18.130:33631
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+--------------------+----------------------------------------
builtin_ntp_servers | 127.8.18.148:42749 | 127.8.18.130:33631, 127.8.18.129:40911
time_source | builtin | 127.8.18.130:33631, 127.8.18.129:40911
Version Summary
Version | Servers
-----------------+-----------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.8.18.190:40323, tserver@127.8.18.130:33631, tserver@127.8.18.129:40911
Tablet Summary
Tablet 1f99182d93ed483e9f520bf312a591e3 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
All reported replicas are:
A = d06d8f442c824758ae944ffe71e3cb44
B = 01cff8a283e94658aea5fad9fa0b2db4
C = 747982f4b834445b93f077dd1295b2ff
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | -1 | Yes
C | A B* C | 1 | -1 | Yes
Tablet 459e1d55c65e4857a7ccba6b13316243 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet b29f930721b943039d1df1f03e073a74 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | -1 | Yes
B | A B* C | 2 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:29:06.239945 8266 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
I20260810 22:29:07.300539 8670 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20260810 22:29:07.300602 8486 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+--------------------+---------
e52d1e743df1434f88d46b1016be2ae7 | 127.8.18.190:40323 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.8.18.148:42749 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+--------------------+-------------+----------+----------------+-----------------
01cff8a283e94658aea5fad9fa0b2db4 | 127.8.18.130:33631 | HEALTHY | <none> | 3 | 0
747982f4b834445b93f077dd1295b2ff | 127.8.18.129:40911 | HEALTHY | <none> | 0 | 0
d06d8f442c824758ae944ffe71e3cb44 | 127.8.18.131:36507 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.8.18.131:36507: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+----------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.8.18.130:33631, 127.8.18.129:40911
ipki_server_key_size | 768 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.129 | experimental | 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.130 | experimental | 127.8.18.130:33631
never_fsync | true | unsafe,advanced | 127.8.18.130:33631, 127.8.18.129:40911
openssl_security_level_override | 0 | unsafe,hidden | 127.8.18.130:33631, 127.8.18.129:40911
rpc_server_allow_ephemeral_ports | true | unsafe | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_format | pb | hidden | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-0/data/info.pb | hidden | 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-1/data/info.pb | hidden | 127.8.18.130:33631
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+--------------------+----------------------------------------
builtin_ntp_servers | 127.8.18.148:42749 | 127.8.18.130:33631, 127.8.18.129:40911
time_source | builtin | 127.8.18.130:33631, 127.8.18.129:40911
Version Summary
Version | Servers
-----------------+-----------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.8.18.190:40323, tserver@127.8.18.130:33631, tserver@127.8.18.129:40911
Tablet Summary
Tablet 1f99182d93ed483e9f520bf312a591e3 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
All reported replicas are:
A = d06d8f442c824758ae944ffe71e3cb44
B = 01cff8a283e94658aea5fad9fa0b2db4
C = 747982f4b834445b93f077dd1295b2ff
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | -1 | Yes
C | A B* C | 1 | -1 | Yes
Tablet 459e1d55c65e4857a7ccba6b13316243 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet b29f930721b943039d1df1f03e073a74 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | -1 | Yes
B | A B* C | 2 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:29:07.329039 8266 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:29:07.360618 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 56: this message will repeat every 5th retry.
W20260810 22:29:08.097270 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 61: this message will repeat every 5th retry.
I20260810 22:29:08.381460 8670 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20260810 22:29:08.382761 8486 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+--------------------+---------
e52d1e743df1434f88d46b1016be2ae7 | 127.8.18.190:40323 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.8.18.148:42749 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+--------------------+-------------+----------+----------------+-----------------
01cff8a283e94658aea5fad9fa0b2db4 | 127.8.18.130:33631 | HEALTHY | <none> | 3 | 0
747982f4b834445b93f077dd1295b2ff | 127.8.18.129:40911 | HEALTHY | <none> | 0 | 0
d06d8f442c824758ae944ffe71e3cb44 | 127.8.18.131:36507 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.8.18.131:36507: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+----------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.8.18.130:33631, 127.8.18.129:40911
ipki_server_key_size | 768 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.129 | experimental | 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.130 | experimental | 127.8.18.130:33631
never_fsync | true | unsafe,advanced | 127.8.18.130:33631, 127.8.18.129:40911
openssl_security_level_override | 0 | unsafe,hidden | 127.8.18.130:33631, 127.8.18.129:40911
rpc_server_allow_ephemeral_ports | true | unsafe | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_format | pb | hidden | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-0/data/info.pb | hidden | 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-1/data/info.pb | hidden | 127.8.18.130:33631
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+--------------------+----------------------------------------
builtin_ntp_servers | 127.8.18.148:42749 | 127.8.18.130:33631, 127.8.18.129:40911
time_source | builtin | 127.8.18.130:33631, 127.8.18.129:40911
Version Summary
Version | Servers
-----------------+-----------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.8.18.190:40323, tserver@127.8.18.130:33631, tserver@127.8.18.129:40911
Tablet Summary
Tablet 1f99182d93ed483e9f520bf312a591e3 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
All reported replicas are:
A = d06d8f442c824758ae944ffe71e3cb44
B = 01cff8a283e94658aea5fad9fa0b2db4
C = 747982f4b834445b93f077dd1295b2ff
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | -1 | Yes
C | A B* C | 1 | -1 | Yes
Tablet 459e1d55c65e4857a7ccba6b13316243 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet b29f930721b943039d1df1f03e073a74 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | -1 | Yes
B | A B* C | 2 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:29:08.402704 8266 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:29:08.429466 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 61: this message will repeat every 5th retry.
W20260810 22:29:09.248418 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 31 similar messages]
I20260810 22:29:09.459697 8486 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20260810 22:29:09.460330 8670 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+--------------------+---------
e52d1e743df1434f88d46b1016be2ae7 | 127.8.18.190:40323 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.8.18.148:42749 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+--------------------+-------------+----------+----------------+-----------------
01cff8a283e94658aea5fad9fa0b2db4 | 127.8.18.130:33631 | HEALTHY | <none> | 3 | 0
747982f4b834445b93f077dd1295b2ff | 127.8.18.129:40911 | HEALTHY | <none> | 0 | 0
d06d8f442c824758ae944ffe71e3cb44 | 127.8.18.131:36507 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.8.18.131:36507: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+----------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.8.18.130:33631, 127.8.18.129:40911
ipki_server_key_size | 768 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.129 | experimental | 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.130 | experimental | 127.8.18.130:33631
never_fsync | true | unsafe,advanced | 127.8.18.130:33631, 127.8.18.129:40911
openssl_security_level_override | 0 | unsafe,hidden | 127.8.18.130:33631, 127.8.18.129:40911
rpc_server_allow_ephemeral_ports | true | unsafe | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_format | pb | hidden | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-0/data/info.pb | hidden | 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-1/data/info.pb | hidden | 127.8.18.130:33631
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+--------------------+----------------------------------------
builtin_ntp_servers | 127.8.18.148:42749 | 127.8.18.130:33631, 127.8.18.129:40911
time_source | builtin | 127.8.18.130:33631, 127.8.18.129:40911
Version Summary
Version | Servers
-----------------+-----------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.8.18.190:40323, tserver@127.8.18.129:40911, tserver@127.8.18.130:33631
Tablet Summary
Tablet 1f99182d93ed483e9f520bf312a591e3 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
All reported replicas are:
A = d06d8f442c824758ae944ffe71e3cb44
B = 01cff8a283e94658aea5fad9fa0b2db4
C = 747982f4b834445b93f077dd1295b2ff
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | -1 | Yes
C | A B* C | 1 | -1 | Yes
Tablet 459e1d55c65e4857a7ccba6b13316243 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet b29f930721b943039d1df1f03e073a74 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | -1 | Yes
B | A B* C | 2 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:29:09.481016 8266 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:29:09.808005 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 61: this message will repeat every 5th retry.
I20260810 22:29:10.537222 8670 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20260810 22:29:10.537222 8486 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+--------------------+---------
e52d1e743df1434f88d46b1016be2ae7 | 127.8.18.190:40323 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.8.18.148:42749 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+--------------------+-------------+----------+----------------+-----------------
01cff8a283e94658aea5fad9fa0b2db4 | 127.8.18.130:33631 | HEALTHY | <none> | 3 | 0
747982f4b834445b93f077dd1295b2ff | 127.8.18.129:40911 | HEALTHY | <none> | 0 | 0
d06d8f442c824758ae944ffe71e3cb44 | 127.8.18.131:36507 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.8.18.131:36507: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+----------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.8.18.130:33631, 127.8.18.129:40911
ipki_server_key_size | 768 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.129 | experimental | 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.130 | experimental | 127.8.18.130:33631
never_fsync | true | unsafe,advanced | 127.8.18.130:33631, 127.8.18.129:40911
openssl_security_level_override | 0 | unsafe,hidden | 127.8.18.130:33631, 127.8.18.129:40911
rpc_server_allow_ephemeral_ports | true | unsafe | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_format | pb | hidden | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-0/data/info.pb | hidden | 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-1/data/info.pb | hidden | 127.8.18.130:33631
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+--------------------+----------------------------------------
builtin_ntp_servers | 127.8.18.148:42749 | 127.8.18.130:33631, 127.8.18.129:40911
time_source | builtin | 127.8.18.130:33631, 127.8.18.129:40911
Version Summary
Version | Servers
-----------------+-----------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.8.18.190:40323, tserver@127.8.18.129:40911, tserver@127.8.18.130:33631
Tablet Summary
Tablet 1f99182d93ed483e9f520bf312a591e3 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
All reported replicas are:
A = d06d8f442c824758ae944ffe71e3cb44
B = 01cff8a283e94658aea5fad9fa0b2db4
C = 747982f4b834445b93f077dd1295b2ff
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | -1 | Yes
C | A B* C | 1 | -1 | Yes
Tablet 459e1d55c65e4857a7ccba6b13316243 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet b29f930721b943039d1df1f03e073a74 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | -1 | Yes
B | A B* C | 2 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:29:10.564669 8266 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:29:10.577839 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 66: this message will repeat every 5th retry.
W20260810 22:29:10.989401 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 66: this message will repeat every 5th retry.
I20260810 22:29:11.619911 8670 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20260810 22:29:11.620139 8486 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+--------------------+---------
e52d1e743df1434f88d46b1016be2ae7 | 127.8.18.190:40323 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.8.18.148:42749 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+--------------------+-------------+----------+----------------+-----------------
01cff8a283e94658aea5fad9fa0b2db4 | 127.8.18.130:33631 | HEALTHY | <none> | 3 | 0
747982f4b834445b93f077dd1295b2ff | 127.8.18.129:40911 | HEALTHY | <none> | 0 | 0
d06d8f442c824758ae944ffe71e3cb44 | 127.8.18.131:36507 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.8.18.131:36507: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+----------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.8.18.130:33631, 127.8.18.129:40911
ipki_server_key_size | 768 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.129 | experimental | 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.130 | experimental | 127.8.18.130:33631
never_fsync | true | unsafe,advanced | 127.8.18.130:33631, 127.8.18.129:40911
openssl_security_level_override | 0 | unsafe,hidden | 127.8.18.130:33631, 127.8.18.129:40911
rpc_server_allow_ephemeral_ports | true | unsafe | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_format | pb | hidden | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-0/data/info.pb | hidden | 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-1/data/info.pb | hidden | 127.8.18.130:33631
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+--------------------+----------------------------------------
builtin_ntp_servers | 127.8.18.148:42749 | 127.8.18.130:33631, 127.8.18.129:40911
time_source | builtin | 127.8.18.130:33631, 127.8.18.129:40911
Version Summary
Version | Servers
-----------------+-----------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.8.18.190:40323, tserver@127.8.18.130:33631, tserver@127.8.18.129:40911
Tablet Summary
Tablet 1f99182d93ed483e9f520bf312a591e3 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
All reported replicas are:
A = d06d8f442c824758ae944ffe71e3cb44
B = 01cff8a283e94658aea5fad9fa0b2db4
C = 747982f4b834445b93f077dd1295b2ff
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | -1 | Yes
C | A B* C | 1 | -1 | Yes
Tablet 459e1d55c65e4857a7ccba6b13316243 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet b29f930721b943039d1df1f03e073a74 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | -1 | Yes
B | A B* C | 2 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:29:11.645248 8266 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:29:12.430770 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 66: this message will repeat every 5th retry.
I20260810 22:29:12.706542 8486 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20260810 22:29:12.706857 8670 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+--------------------+---------
e52d1e743df1434f88d46b1016be2ae7 | 127.8.18.190:40323 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.8.18.148:42749 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+--------------------+-------------+----------+----------------+-----------------
01cff8a283e94658aea5fad9fa0b2db4 | 127.8.18.130:33631 | HEALTHY | <none> | 3 | 0
747982f4b834445b93f077dd1295b2ff | 127.8.18.129:40911 | HEALTHY | <none> | 0 | 0
d06d8f442c824758ae944ffe71e3cb44 | 127.8.18.131:36507 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.8.18.131:36507: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+----------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.8.18.130:33631, 127.8.18.129:40911
ipki_server_key_size | 768 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.129 | experimental | 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.130 | experimental | 127.8.18.130:33631
never_fsync | true | unsafe,advanced | 127.8.18.130:33631, 127.8.18.129:40911
openssl_security_level_override | 0 | unsafe,hidden | 127.8.18.130:33631, 127.8.18.129:40911
rpc_server_allow_ephemeral_ports | true | unsafe | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_format | pb | hidden | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-0/data/info.pb | hidden | 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-1/data/info.pb | hidden | 127.8.18.130:33631
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+--------------------+----------------------------------------
builtin_ntp_servers | 127.8.18.148:42749 | 127.8.18.130:33631, 127.8.18.129:40911
time_source | builtin | 127.8.18.130:33631, 127.8.18.129:40911
Version Summary
Version | Servers
-----------------+-----------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.8.18.190:40323, tserver@127.8.18.129:40911, tserver@127.8.18.130:33631
Tablet Summary
Tablet 1f99182d93ed483e9f520bf312a591e3 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
All reported replicas are:
A = d06d8f442c824758ae944ffe71e3cb44
B = 01cff8a283e94658aea5fad9fa0b2db4
C = 747982f4b834445b93f077dd1295b2ff
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | -1 | Yes
C | A B* C | 1 | -1 | Yes
Tablet 459e1d55c65e4857a7ccba6b13316243 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet b29f930721b943039d1df1f03e073a74 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | -1 | Yes
B | A B* C | 2 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:29:12.726167 8266 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:29:13.099273 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 71: this message will repeat every 5th retry.
W20260810 22:29:13.598934 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 71: this message will repeat every 5th retry.
I20260810 22:29:13.779676 8670 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20260810 22:29:13.780699 8486 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+--------------------+---------
e52d1e743df1434f88d46b1016be2ae7 | 127.8.18.190:40323 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.8.18.148:42749 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+--------------------+-------------+----------+----------------+-----------------
01cff8a283e94658aea5fad9fa0b2db4 | 127.8.18.130:33631 | HEALTHY | <none> | 3 | 0
747982f4b834445b93f077dd1295b2ff | 127.8.18.129:40911 | HEALTHY | <none> | 0 | 0
d06d8f442c824758ae944ffe71e3cb44 | 127.8.18.131:36507 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.8.18.131:36507: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+----------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.8.18.130:33631, 127.8.18.129:40911
ipki_server_key_size | 768 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.129 | experimental | 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.130 | experimental | 127.8.18.130:33631
never_fsync | true | unsafe,advanced | 127.8.18.130:33631, 127.8.18.129:40911
openssl_security_level_override | 0 | unsafe,hidden | 127.8.18.130:33631, 127.8.18.129:40911
rpc_server_allow_ephemeral_ports | true | unsafe | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_format | pb | hidden | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-0/data/info.pb | hidden | 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-1/data/info.pb | hidden | 127.8.18.130:33631
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+--------------------+----------------------------------------
builtin_ntp_servers | 127.8.18.148:42749 | 127.8.18.130:33631, 127.8.18.129:40911
time_source | builtin | 127.8.18.130:33631, 127.8.18.129:40911
Version Summary
Version | Servers
-----------------+-----------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.8.18.190:40323, tserver@127.8.18.130:33631, tserver@127.8.18.129:40911
Tablet Summary
Tablet 1f99182d93ed483e9f520bf312a591e3 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
All reported replicas are:
A = d06d8f442c824758ae944ffe71e3cb44
B = 01cff8a283e94658aea5fad9fa0b2db4
C = 747982f4b834445b93f077dd1295b2ff
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | -1 | Yes
C | A B* C | 1 | -1 | Yes
Tablet 459e1d55c65e4857a7ccba6b13316243 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet b29f930721b943039d1df1f03e073a74 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | -1 | Yes
B | A B* C | 2 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:29:13.800666 8266 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:29:14.445322 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 29 similar messages]
I20260810 22:29:14.854712 8486 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20260810 22:29:14.854677 8670 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
W20260810 22:29:14.868110 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 71: this message will repeat every 5th retry.
Master Summary
UUID | Address | Status
----------------------------------+--------------------+---------
e52d1e743df1434f88d46b1016be2ae7 | 127.8.18.190:40323 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.8.18.148:42749 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+--------------------+-------------+----------+----------------+-----------------
01cff8a283e94658aea5fad9fa0b2db4 | 127.8.18.130:33631 | HEALTHY | <none> | 3 | 0
747982f4b834445b93f077dd1295b2ff | 127.8.18.129:40911 | HEALTHY | <none> | 0 | 0
d06d8f442c824758ae944ffe71e3cb44 | 127.8.18.131:36507 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.8.18.131:36507: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+----------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.8.18.130:33631, 127.8.18.129:40911
ipki_server_key_size | 768 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.129 | experimental | 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.130 | experimental | 127.8.18.130:33631
never_fsync | true | unsafe,advanced | 127.8.18.130:33631, 127.8.18.129:40911
openssl_security_level_override | 0 | unsafe,hidden | 127.8.18.130:33631, 127.8.18.129:40911
rpc_server_allow_ephemeral_ports | true | unsafe | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_format | pb | hidden | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-0/data/info.pb | hidden | 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-1/data/info.pb | hidden | 127.8.18.130:33631
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+--------------------+----------------------------------------
builtin_ntp_servers | 127.8.18.148:42749 | 127.8.18.130:33631, 127.8.18.129:40911
time_source | builtin | 127.8.18.130:33631, 127.8.18.129:40911
Version Summary
Version | Servers
-----------------+-----------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.8.18.190:40323, tserver@127.8.18.130:33631, tserver@127.8.18.129:40911
Tablet Summary
Tablet 1f99182d93ed483e9f520bf312a591e3 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
All reported replicas are:
A = d06d8f442c824758ae944ffe71e3cb44
B = 01cff8a283e94658aea5fad9fa0b2db4
C = 747982f4b834445b93f077dd1295b2ff
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | -1 | Yes
C | A B* C | 1 | -1 | Yes
Tablet 459e1d55c65e4857a7ccba6b13316243 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet b29f930721b943039d1df1f03e073a74 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | -1 | Yes
B | A B* C | 2 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:29:14.875620 8266 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:29:15.646600 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 76: this message will repeat every 5th retry.
I20260810 22:29:15.931913 8486 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20260810 22:29:15.932971 8670 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+--------------------+---------
e52d1e743df1434f88d46b1016be2ae7 | 127.8.18.190:40323 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.8.18.148:42749 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+--------------------+-------------+----------+----------------+-----------------
01cff8a283e94658aea5fad9fa0b2db4 | 127.8.18.130:33631 | HEALTHY | <none> | 3 | 0
747982f4b834445b93f077dd1295b2ff | 127.8.18.129:40911 | HEALTHY | <none> | 0 | 0
d06d8f442c824758ae944ffe71e3cb44 | 127.8.18.131:36507 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.8.18.131:36507: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+----------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.8.18.130:33631, 127.8.18.129:40911
ipki_server_key_size | 768 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.129 | experimental | 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.130 | experimental | 127.8.18.130:33631
never_fsync | true | unsafe,advanced | 127.8.18.130:33631, 127.8.18.129:40911
openssl_security_level_override | 0 | unsafe,hidden | 127.8.18.130:33631, 127.8.18.129:40911
rpc_server_allow_ephemeral_ports | true | unsafe | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_format | pb | hidden | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-0/data/info.pb | hidden | 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-1/data/info.pb | hidden | 127.8.18.130:33631
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+--------------------+----------------------------------------
builtin_ntp_servers | 127.8.18.148:42749 | 127.8.18.130:33631, 127.8.18.129:40911
time_source | builtin | 127.8.18.130:33631, 127.8.18.129:40911
Version Summary
Version | Servers
-----------------+-----------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.8.18.190:40323, tserver@127.8.18.129:40911, tserver@127.8.18.130:33631
Tablet Summary
Tablet 1f99182d93ed483e9f520bf312a591e3 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
All reported replicas are:
A = d06d8f442c824758ae944ffe71e3cb44
B = 01cff8a283e94658aea5fad9fa0b2db4
C = 747982f4b834445b93f077dd1295b2ff
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | -1 | Yes
C | A B* C | 1 | -1 | Yes
Tablet 459e1d55c65e4857a7ccba6b13316243 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet b29f930721b943039d1df1f03e073a74 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | -1 | Yes
B | A B* C | 2 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:29:15.952011 8266 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:29:16.416529 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 76: this message will repeat every 5th retry.
I20260810 22:29:17.007212 8670 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20260810 22:29:17.007448 8486 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+--------------------+---------
e52d1e743df1434f88d46b1016be2ae7 | 127.8.18.190:40323 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.8.18.148:42749 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+--------------------+-------------+----------+----------------+-----------------
01cff8a283e94658aea5fad9fa0b2db4 | 127.8.18.130:33631 | HEALTHY | <none> | 3 | 0
747982f4b834445b93f077dd1295b2ff | 127.8.18.129:40911 | HEALTHY | <none> | 0 | 0
d06d8f442c824758ae944ffe71e3cb44 | 127.8.18.131:36507 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.8.18.131:36507: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+----------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.8.18.130:33631, 127.8.18.129:40911
ipki_server_key_size | 768 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.129 | experimental | 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.130 | experimental | 127.8.18.130:33631
never_fsync | true | unsafe,advanced | 127.8.18.130:33631, 127.8.18.129:40911
openssl_security_level_override | 0 | unsafe,hidden | 127.8.18.130:33631, 127.8.18.129:40911
rpc_server_allow_ephemeral_ports | true | unsafe | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_format | pb | hidden | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-0/data/info.pb | hidden | 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-1/data/info.pb | hidden | 127.8.18.130:33631
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+--------------------+----------------------------------------
builtin_ntp_servers | 127.8.18.148:42749 | 127.8.18.130:33631, 127.8.18.129:40911
time_source | builtin | 127.8.18.130:33631, 127.8.18.129:40911
Version Summary
Version | Servers
-----------------+-----------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.8.18.190:40323, tserver@127.8.18.130:33631, tserver@127.8.18.129:40911
Tablet Summary
Tablet 1f99182d93ed483e9f520bf312a591e3 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
All reported replicas are:
A = d06d8f442c824758ae944ffe71e3cb44
B = 01cff8a283e94658aea5fad9fa0b2db4
C = 747982f4b834445b93f077dd1295b2ff
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | -1 | Yes
C | A B* C | 1 | -1 | Yes
Tablet 459e1d55c65e4857a7ccba6b13316243 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet b29f930721b943039d1df1f03e073a74 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | -1 | Yes
B | A B* C | 2 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:29:17.027277 8266 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:29:17.511289 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 76: this message will repeat every 5th retry.
I20260810 22:29:18.087677 8670 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20260810 22:29:18.087984 8486 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+--------------------+---------
e52d1e743df1434f88d46b1016be2ae7 | 127.8.18.190:40323 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.8.18.148:42749 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+--------------------+-------------+----------+----------------+-----------------
01cff8a283e94658aea5fad9fa0b2db4 | 127.8.18.130:33631 | HEALTHY | <none> | 3 | 0
747982f4b834445b93f077dd1295b2ff | 127.8.18.129:40911 | HEALTHY | <none> | 0 | 0
d06d8f442c824758ae944ffe71e3cb44 | 127.8.18.131:36507 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.8.18.131:36507: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+----------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.8.18.130:33631, 127.8.18.129:40911
ipki_server_key_size | 768 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.129 | experimental | 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.130 | experimental | 127.8.18.130:33631
never_fsync | true | unsafe,advanced | 127.8.18.130:33631, 127.8.18.129:40911
openssl_security_level_override | 0 | unsafe,hidden | 127.8.18.130:33631, 127.8.18.129:40911
rpc_server_allow_ephemeral_ports | true | unsafe | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_format | pb | hidden | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-0/data/info.pb | hidden | 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-1/data/info.pb | hidden | 127.8.18.130:33631
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+--------------------+----------------------------------------
builtin_ntp_servers | 127.8.18.148:42749 | 127.8.18.130:33631, 127.8.18.129:40911
time_source | builtin | 127.8.18.130:33631, 127.8.18.129:40911
Version Summary
Version | Servers
-----------------+-----------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.8.18.190:40323, tserver@127.8.18.130:33631, tserver@127.8.18.129:40911
Tablet Summary
Tablet 1f99182d93ed483e9f520bf312a591e3 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
All reported replicas are:
A = d06d8f442c824758ae944ffe71e3cb44
B = 01cff8a283e94658aea5fad9fa0b2db4
C = 747982f4b834445b93f077dd1295b2ff
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | -1 | Yes
C | A B* C | 1 | -1 | Yes
Tablet 459e1d55c65e4857a7ccba6b13316243 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet b29f930721b943039d1df1f03e073a74 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | -1 | Yes
B | A B* C | 2 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:29:18.108812 8266 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:29:18.203807 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 81: this message will repeat every 5th retry.
W20260810 22:29:19.143764 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 81: this message will repeat every 5th retry.
I20260810 22:29:19.163185 8670 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20260810 22:29:19.163523 8486 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+--------------------+---------
e52d1e743df1434f88d46b1016be2ae7 | 127.8.18.190:40323 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.8.18.148:42749 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+--------------------+-------------+----------+----------------+-----------------
01cff8a283e94658aea5fad9fa0b2db4 | 127.8.18.130:33631 | HEALTHY | <none> | 3 | 0
747982f4b834445b93f077dd1295b2ff | 127.8.18.129:40911 | HEALTHY | <none> | 0 | 0
d06d8f442c824758ae944ffe71e3cb44 | 127.8.18.131:36507 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.8.18.131:36507: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+----------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.8.18.130:33631, 127.8.18.129:40911
ipki_server_key_size | 768 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.129 | experimental | 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.130 | experimental | 127.8.18.130:33631
never_fsync | true | unsafe,advanced | 127.8.18.130:33631, 127.8.18.129:40911
openssl_security_level_override | 0 | unsafe,hidden | 127.8.18.130:33631, 127.8.18.129:40911
rpc_server_allow_ephemeral_ports | true | unsafe | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_format | pb | hidden | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-0/data/info.pb | hidden | 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-1/data/info.pb | hidden | 127.8.18.130:33631
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+--------------------+----------------------------------------
builtin_ntp_servers | 127.8.18.148:42749 | 127.8.18.130:33631, 127.8.18.129:40911
time_source | builtin | 127.8.18.130:33631, 127.8.18.129:40911
Version Summary
Version | Servers
-----------------+-----------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.8.18.190:40323, tserver@127.8.18.130:33631, tserver@127.8.18.129:40911
Tablet Summary
Tablet 1f99182d93ed483e9f520bf312a591e3 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
All reported replicas are:
A = d06d8f442c824758ae944ffe71e3cb44
B = 01cff8a283e94658aea5fad9fa0b2db4
C = 747982f4b834445b93f077dd1295b2ff
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | -1 | Yes
C | A B* C | 1 | -1 | Yes
Tablet 459e1d55c65e4857a7ccba6b13316243 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet b29f930721b943039d1df1f03e073a74 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | -1 | Yes
B | A B* C | 2 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:29:19.183843 8266 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:29:19.626420 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 29 similar messages]
W20260810 22:29:19.900770 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 81: this message will repeat every 5th retry.
I20260810 22:29:20.238906 8670 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20260810 22:29:20.239192 8486 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+--------------------+---------
e52d1e743df1434f88d46b1016be2ae7 | 127.8.18.190:40323 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.8.18.148:42749 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+--------------------+-------------+----------+----------------+-----------------
01cff8a283e94658aea5fad9fa0b2db4 | 127.8.18.130:33631 | HEALTHY | <none> | 3 | 0
747982f4b834445b93f077dd1295b2ff | 127.8.18.129:40911 | HEALTHY | <none> | 0 | 0
d06d8f442c824758ae944ffe71e3cb44 | 127.8.18.131:36507 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.8.18.131:36507: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+----------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.8.18.130:33631, 127.8.18.129:40911
ipki_server_key_size | 768 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.129 | experimental | 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.130 | experimental | 127.8.18.130:33631
never_fsync | true | unsafe,advanced | 127.8.18.130:33631, 127.8.18.129:40911
openssl_security_level_override | 0 | unsafe,hidden | 127.8.18.130:33631, 127.8.18.129:40911
rpc_server_allow_ephemeral_ports | true | unsafe | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_format | pb | hidden | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-0/data/info.pb | hidden | 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-1/data/info.pb | hidden | 127.8.18.130:33631
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+--------------------+----------------------------------------
builtin_ntp_servers | 127.8.18.148:42749 | 127.8.18.130:33631, 127.8.18.129:40911
time_source | builtin | 127.8.18.130:33631, 127.8.18.129:40911
Version Summary
Version | Servers
-----------------+-----------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.8.18.190:40323, tserver@127.8.18.130:33631, tserver@127.8.18.129:40911
Tablet Summary
Tablet 1f99182d93ed483e9f520bf312a591e3 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
All reported replicas are:
A = d06d8f442c824758ae944ffe71e3cb44
B = 01cff8a283e94658aea5fad9fa0b2db4
C = 747982f4b834445b93f077dd1295b2ff
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | -1 | Yes
C | A B* C | 1 | -1 | Yes
Tablet 459e1d55c65e4857a7ccba6b13316243 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet b29f930721b943039d1df1f03e073a74 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | -1 | Yes
B | A B* C | 2 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:29:20.262005 8266 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:29:20.605422 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 86: this message will repeat every 5th retry.
I20260810 22:29:21.317293 8670 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20260810 22:29:21.317414 8486 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+--------------------+---------
e52d1e743df1434f88d46b1016be2ae7 | 127.8.18.190:40323 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.8.18.148:42749 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+--------------------+-------------+----------+----------------+-----------------
01cff8a283e94658aea5fad9fa0b2db4 | 127.8.18.130:33631 | HEALTHY | <none> | 3 | 0
747982f4b834445b93f077dd1295b2ff | 127.8.18.129:40911 | HEALTHY | <none> | 0 | 0
d06d8f442c824758ae944ffe71e3cb44 | 127.8.18.131:36507 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.8.18.131:36507: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+----------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.8.18.130:33631, 127.8.18.129:40911
ipki_server_key_size | 768 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.129 | experimental | 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.130 | experimental | 127.8.18.130:33631
never_fsync | true | unsafe,advanced | 127.8.18.130:33631, 127.8.18.129:40911
openssl_security_level_override | 0 | unsafe,hidden | 127.8.18.130:33631, 127.8.18.129:40911
rpc_server_allow_ephemeral_ports | true | unsafe | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_format | pb | hidden | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-0/data/info.pb | hidden | 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-1/data/info.pb | hidden | 127.8.18.130:33631
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+--------------------+----------------------------------------
builtin_ntp_servers | 127.8.18.148:42749 | 127.8.18.130:33631, 127.8.18.129:40911
time_source | builtin | 127.8.18.130:33631, 127.8.18.129:40911
Version Summary
Version | Servers
-----------------+-----------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.8.18.190:40323, tserver@127.8.18.130:33631, tserver@127.8.18.129:40911
Tablet Summary
Tablet 1f99182d93ed483e9f520bf312a591e3 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
All reported replicas are:
A = d06d8f442c824758ae944ffe71e3cb44
B = 01cff8a283e94658aea5fad9fa0b2db4
C = 747982f4b834445b93f077dd1295b2ff
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | -1 | Yes
C | A B* C | 1 | -1 | Yes
Tablet 459e1d55c65e4857a7ccba6b13316243 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet b29f930721b943039d1df1f03e073a74 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | -1 | Yes
B | A B* C | 2 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:29:21.339787 8266 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:29:21.566228 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 86: this message will repeat every 5th retry.
I20260810 22:29:22.394122 8486 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20260810 22:29:22.394205 8670 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+--------------------+---------
e52d1e743df1434f88d46b1016be2ae7 | 127.8.18.190:40323 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.8.18.148:42749 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+--------------------+-------------+----------+----------------+-----------------
01cff8a283e94658aea5fad9fa0b2db4 | 127.8.18.130:33631 | HEALTHY | <none> | 3 | 0
747982f4b834445b93f077dd1295b2ff | 127.8.18.129:40911 | HEALTHY | <none> | 0 | 0
d06d8f442c824758ae944ffe71e3cb44 | 127.8.18.131:36507 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.8.18.131:36507: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+----------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.8.18.130:33631, 127.8.18.129:40911
ipki_server_key_size | 768 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.129 | experimental | 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.130 | experimental | 127.8.18.130:33631
never_fsync | true | unsafe,advanced | 127.8.18.130:33631, 127.8.18.129:40911
openssl_security_level_override | 0 | unsafe,hidden | 127.8.18.130:33631, 127.8.18.129:40911
rpc_server_allow_ephemeral_ports | true | unsafe | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_format | pb | hidden | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-0/data/info.pb | hidden | 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-1/data/info.pb | hidden | 127.8.18.130:33631
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+--------------------+----------------------------------------
builtin_ntp_servers | 127.8.18.148:42749 | 127.8.18.130:33631, 127.8.18.129:40911
time_source | builtin | 127.8.18.130:33631, 127.8.18.129:40911
Version Summary
Version | Servers
-----------------+-----------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.8.18.190:40323, tserver@127.8.18.130:33631, tserver@127.8.18.129:40911
Tablet Summary
Tablet 1f99182d93ed483e9f520bf312a591e3 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
All reported replicas are:
A = d06d8f442c824758ae944ffe71e3cb44
B = 01cff8a283e94658aea5fad9fa0b2db4
C = 747982f4b834445b93f077dd1295b2ff
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | -1 | Yes
C | A B* C | 1 | -1 | Yes
Tablet 459e1d55c65e4857a7ccba6b13316243 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet b29f930721b943039d1df1f03e073a74 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | -1 | Yes
B | A B* C | 2 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:29:22.412344 8266 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:29:22.510299 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 86: this message will repeat every 5th retry.
W20260810 22:29:22.921442 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 91: this message will repeat every 5th retry.
I20260810 22:29:23.472311 8670 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20260810 22:29:23.472379 8486 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+--------------------+---------
e52d1e743df1434f88d46b1016be2ae7 | 127.8.18.190:40323 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.8.18.148:42749 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+--------------------+-------------+----------+----------------+-----------------
01cff8a283e94658aea5fad9fa0b2db4 | 127.8.18.130:33631 | HEALTHY | <none> | 3 | 0
747982f4b834445b93f077dd1295b2ff | 127.8.18.129:40911 | HEALTHY | <none> | 0 | 0
d06d8f442c824758ae944ffe71e3cb44 | 127.8.18.131:36507 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.8.18.131:36507: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+----------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.8.18.130:33631, 127.8.18.129:40911
ipki_server_key_size | 768 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.129 | experimental | 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.130 | experimental | 127.8.18.130:33631
never_fsync | true | unsafe,advanced | 127.8.18.130:33631, 127.8.18.129:40911
openssl_security_level_override | 0 | unsafe,hidden | 127.8.18.130:33631, 127.8.18.129:40911
rpc_server_allow_ephemeral_ports | true | unsafe | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_format | pb | hidden | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-0/data/info.pb | hidden | 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-1/data/info.pb | hidden | 127.8.18.130:33631
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+--------------------+----------------------------------------
builtin_ntp_servers | 127.8.18.148:42749 | 127.8.18.130:33631, 127.8.18.129:40911
time_source | builtin | 127.8.18.130:33631, 127.8.18.129:40911
Version Summary
Version | Servers
-----------------+-----------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.8.18.190:40323, tserver@127.8.18.130:33631, tserver@127.8.18.129:40911
Tablet Summary
Tablet 1f99182d93ed483e9f520bf312a591e3 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
All reported replicas are:
A = d06d8f442c824758ae944ffe71e3cb44
B = 01cff8a283e94658aea5fad9fa0b2db4
C = 747982f4b834445b93f077dd1295b2ff
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | -1 | Yes
C | A B* C | 1 | -1 | Yes
Tablet 459e1d55c65e4857a7ccba6b13316243 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet b29f930721b943039d1df1f03e073a74 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | -1 | Yes
B | A B* C | 2 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:29:23.491963 8266 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:29:23.877746 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 91: this message will repeat every 5th retry.
I20260810 22:29:24.545646 8670 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20260810 22:29:24.545724 8486 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+--------------------+---------
e52d1e743df1434f88d46b1016be2ae7 | 127.8.18.190:40323 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.8.18.148:42749 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+--------------------+-------------+----------+----------------+-----------------
01cff8a283e94658aea5fad9fa0b2db4 | 127.8.18.130:33631 | HEALTHY | <none> | 3 | 0
747982f4b834445b93f077dd1295b2ff | 127.8.18.129:40911 | HEALTHY | <none> | 0 | 0
d06d8f442c824758ae944ffe71e3cb44 | 127.8.18.131:36507 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.8.18.131:36507: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+----------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.8.18.130:33631, 127.8.18.129:40911
ipki_server_key_size | 768 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.129 | experimental | 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.130 | experimental | 127.8.18.130:33631
never_fsync | true | unsafe,advanced | 127.8.18.130:33631, 127.8.18.129:40911
openssl_security_level_override | 0 | unsafe,hidden | 127.8.18.130:33631, 127.8.18.129:40911
rpc_server_allow_ephemeral_ports | true | unsafe | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_format | pb | hidden | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-0/data/info.pb | hidden | 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-1/data/info.pb | hidden | 127.8.18.130:33631
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+--------------------+----------------------------------------
builtin_ntp_servers | 127.8.18.148:42749 | 127.8.18.130:33631, 127.8.18.129:40911
time_source | builtin | 127.8.18.130:33631, 127.8.18.129:40911
Version Summary
Version | Servers
-----------------+-----------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.8.18.190:40323, tserver@127.8.18.130:33631, tserver@127.8.18.129:40911
Tablet Summary
Tablet 1f99182d93ed483e9f520bf312a591e3 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
All reported replicas are:
A = d06d8f442c824758ae944ffe71e3cb44
B = 01cff8a283e94658aea5fad9fa0b2db4
C = 747982f4b834445b93f077dd1295b2ff
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | -1 | Yes
C | A B* C | 1 | -1 | Yes
Tablet 459e1d55c65e4857a7ccba6b13316243 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet b29f930721b943039d1df1f03e073a74 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | -1 | Yes
B | A B* C | 2 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:29:24.572299 8266 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:29:24.647063 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 30 similar messages]
W20260810 22:29:25.201041 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 91: this message will repeat every 5th retry.
W20260810 22:29:25.482741 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 96: this message will repeat every 5th retry.
I20260810 22:29:25.626794 8670 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20260810 22:29:25.627709 8486 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+--------------------+---------
e52d1e743df1434f88d46b1016be2ae7 | 127.8.18.190:40323 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.8.18.148:42749 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+--------------------+-------------+----------+----------------+-----------------
01cff8a283e94658aea5fad9fa0b2db4 | 127.8.18.130:33631 | HEALTHY | <none> | 3 | 0
747982f4b834445b93f077dd1295b2ff | 127.8.18.129:40911 | HEALTHY | <none> | 0 | 0
d06d8f442c824758ae944ffe71e3cb44 | 127.8.18.131:36507 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.8.18.131:36507: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+----------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.8.18.130:33631, 127.8.18.129:40911
ipki_server_key_size | 768 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.129 | experimental | 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.130 | experimental | 127.8.18.130:33631
never_fsync | true | unsafe,advanced | 127.8.18.130:33631, 127.8.18.129:40911
openssl_security_level_override | 0 | unsafe,hidden | 127.8.18.130:33631, 127.8.18.129:40911
rpc_server_allow_ephemeral_ports | true | unsafe | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_format | pb | hidden | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-0/data/info.pb | hidden | 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-1/data/info.pb | hidden | 127.8.18.130:33631
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+--------------------+----------------------------------------
builtin_ntp_servers | 127.8.18.148:42749 | 127.8.18.130:33631, 127.8.18.129:40911
time_source | builtin | 127.8.18.130:33631, 127.8.18.129:40911
Version Summary
Version | Servers
-----------------+-----------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.8.18.190:40323, tserver@127.8.18.130:33631, tserver@127.8.18.129:40911
Tablet Summary
Tablet 1f99182d93ed483e9f520bf312a591e3 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
All reported replicas are:
A = d06d8f442c824758ae944ffe71e3cb44
B = 01cff8a283e94658aea5fad9fa0b2db4
C = 747982f4b834445b93f077dd1295b2ff
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | -1 | Yes
C | A B* C | 1 | -1 | Yes
Tablet 459e1d55c65e4857a7ccba6b13316243 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet b29f930721b943039d1df1f03e073a74 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | -1 | Yes
B | A B* C | 2 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:29:25.648499 8266 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:29:26.458815 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 96: this message will repeat every 5th retry.
I20260810 22:29:26.704226 8670 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20260810 22:29:26.704730 8486 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+--------------------+---------
e52d1e743df1434f88d46b1016be2ae7 | 127.8.18.190:40323 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.8.18.148:42749 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+--------------------+-------------+----------+----------------+-----------------
01cff8a283e94658aea5fad9fa0b2db4 | 127.8.18.130:33631 | HEALTHY | <none> | 3 | 0
747982f4b834445b93f077dd1295b2ff | 127.8.18.129:40911 | HEALTHY | <none> | 0 | 0
d06d8f442c824758ae944ffe71e3cb44 | 127.8.18.131:36507 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.8.18.131:36507: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+----------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.8.18.130:33631, 127.8.18.129:40911
ipki_server_key_size | 768 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.129 | experimental | 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.130 | experimental | 127.8.18.130:33631
never_fsync | true | unsafe,advanced | 127.8.18.130:33631, 127.8.18.129:40911
openssl_security_level_override | 0 | unsafe,hidden | 127.8.18.130:33631, 127.8.18.129:40911
rpc_server_allow_ephemeral_ports | true | unsafe | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_format | pb | hidden | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-0/data/info.pb | hidden | 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-1/data/info.pb | hidden | 127.8.18.130:33631
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+--------------------+----------------------------------------
builtin_ntp_servers | 127.8.18.148:42749 | 127.8.18.130:33631, 127.8.18.129:40911
time_source | builtin | 127.8.18.130:33631, 127.8.18.129:40911
Version Summary
Version | Servers
-----------------+-----------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.8.18.190:40323, tserver@127.8.18.130:33631, tserver@127.8.18.129:40911
Tablet Summary
Tablet 1f99182d93ed483e9f520bf312a591e3 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
All reported replicas are:
A = d06d8f442c824758ae944ffe71e3cb44
B = 01cff8a283e94658aea5fad9fa0b2db4
C = 747982f4b834445b93f077dd1295b2ff
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | -1 | Yes
C | A B* C | 1 | -1 | Yes
Tablet 459e1d55c65e4857a7ccba6b13316243 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet b29f930721b943039d1df1f03e073a74 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | -1 | Yes
B | A B* C | 2 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:29:26.723093 8266 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:29:27.388263 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 96: this message will repeat every 5th retry.
I20260810 22:29:27.778870 8670 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20260810 22:29:27.779240 8486 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+--------------------+---------
e52d1e743df1434f88d46b1016be2ae7 | 127.8.18.190:40323 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.8.18.148:42749 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+--------------------+-------------+----------+----------------+-----------------
01cff8a283e94658aea5fad9fa0b2db4 | 127.8.18.130:33631 | HEALTHY | <none> | 3 | 0
747982f4b834445b93f077dd1295b2ff | 127.8.18.129:40911 | HEALTHY | <none> | 0 | 0
d06d8f442c824758ae944ffe71e3cb44 | 127.8.18.131:36507 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.8.18.131:36507: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+----------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.8.18.130:33631, 127.8.18.129:40911
ipki_server_key_size | 768 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.129 | experimental | 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.130 | experimental | 127.8.18.130:33631
never_fsync | true | unsafe,advanced | 127.8.18.130:33631, 127.8.18.129:40911
openssl_security_level_override | 0 | unsafe,hidden | 127.8.18.130:33631, 127.8.18.129:40911
rpc_server_allow_ephemeral_ports | true | unsafe | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_format | pb | hidden | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-0/data/info.pb | hidden | 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-1/data/info.pb | hidden | 127.8.18.130:33631
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+--------------------+----------------------------------------
builtin_ntp_servers | 127.8.18.148:42749 | 127.8.18.130:33631, 127.8.18.129:40911
time_source | builtin | 127.8.18.130:33631, 127.8.18.129:40911
Version Summary
Version | Servers
-----------------+-----------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.8.18.190:40323, tserver@127.8.18.130:33631, tserver@127.8.18.129:40911
Tablet Summary
Tablet 1f99182d93ed483e9f520bf312a591e3 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
All reported replicas are:
A = d06d8f442c824758ae944ffe71e3cb44
B = 01cff8a283e94658aea5fad9fa0b2db4
C = 747982f4b834445b93f077dd1295b2ff
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | -1 | Yes
C | A B* C | 1 | -1 | Yes
Tablet 459e1d55c65e4857a7ccba6b13316243 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet b29f930721b943039d1df1f03e073a74 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | -1 | Yes
B | A B* C | 2 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:29:27.799599 8266 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:29:27.975836 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 101: this message will repeat every 5th retry.
I20260810 22:29:28.863857 8486 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20260810 22:29:28.863903 8670 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+--------------------+---------
e52d1e743df1434f88d46b1016be2ae7 | 127.8.18.190:40323 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.8.18.148:42749 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+--------------------+-------------+----------+----------------+-----------------
01cff8a283e94658aea5fad9fa0b2db4 | 127.8.18.130:33631 | HEALTHY | <none> | 3 | 0
747982f4b834445b93f077dd1295b2ff | 127.8.18.129:40911 | HEALTHY | <none> | 0 | 0
d06d8f442c824758ae944ffe71e3cb44 | 127.8.18.131:36507 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.8.18.131:36507: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+----------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.8.18.130:33631, 127.8.18.129:40911
ipki_server_key_size | 768 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.129 | experimental | 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.130 | experimental | 127.8.18.130:33631
never_fsync | true | unsafe,advanced | 127.8.18.130:33631, 127.8.18.129:40911
openssl_security_level_override | 0 | unsafe,hidden | 127.8.18.130:33631, 127.8.18.129:40911
rpc_server_allow_ephemeral_ports | true | unsafe | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_format | pb | hidden | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-0/data/info.pb | hidden | 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-1/data/info.pb | hidden | 127.8.18.130:33631
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+--------------------+----------------------------------------
builtin_ntp_servers | 127.8.18.148:42749 | 127.8.18.130:33631, 127.8.18.129:40911
time_source | builtin | 127.8.18.130:33631, 127.8.18.129:40911
Version Summary
Version | Servers
-----------------+-----------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.8.18.190:40323, tserver@127.8.18.129:40911, tserver@127.8.18.130:33631
Tablet Summary
Tablet 1f99182d93ed483e9f520bf312a591e3 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
All reported replicas are:
A = d06d8f442c824758ae944ffe71e3cb44
B = 01cff8a283e94658aea5fad9fa0b2db4
C = 747982f4b834445b93f077dd1295b2ff
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | -1 | Yes
C | A B* C | 1 | -1 | Yes
Tablet 459e1d55c65e4857a7ccba6b13316243 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet b29f930721b943039d1df1f03e073a74 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | -1 | Yes
B | A B* C | 2 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:29:28.885119 8266 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:29:29.219359 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 101: this message will repeat every 5th retry.
W20260810 22:29:29.710938 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 29 similar messages]
W20260810 22:29:29.923447 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 101: this message will repeat every 5th retry.
I20260810 22:29:29.939605 8486 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20260810 22:29:29.939652 8670 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+--------------------+---------
e52d1e743df1434f88d46b1016be2ae7 | 127.8.18.190:40323 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.8.18.148:42749 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+--------------------+-------------+----------+----------------+-----------------
01cff8a283e94658aea5fad9fa0b2db4 | 127.8.18.130:33631 | HEALTHY | <none> | 3 | 0
747982f4b834445b93f077dd1295b2ff | 127.8.18.129:40911 | HEALTHY | <none> | 0 | 0
d06d8f442c824758ae944ffe71e3cb44 | 127.8.18.131:36507 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.8.18.131:36507: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+----------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.8.18.130:33631, 127.8.18.129:40911
ipki_server_key_size | 768 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.129 | experimental | 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.130 | experimental | 127.8.18.130:33631
never_fsync | true | unsafe,advanced | 127.8.18.130:33631, 127.8.18.129:40911
openssl_security_level_override | 0 | unsafe,hidden | 127.8.18.130:33631, 127.8.18.129:40911
rpc_server_allow_ephemeral_ports | true | unsafe | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_format | pb | hidden | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-0/data/info.pb | hidden | 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-1/data/info.pb | hidden | 127.8.18.130:33631
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+--------------------+----------------------------------------
builtin_ntp_servers | 127.8.18.148:42749 | 127.8.18.130:33631, 127.8.18.129:40911
time_source | builtin | 127.8.18.130:33631, 127.8.18.129:40911
Version Summary
Version | Servers
-----------------+-----------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.8.18.190:40323, tserver@127.8.18.130:33631, tserver@127.8.18.129:40911
Tablet Summary
Tablet 1f99182d93ed483e9f520bf312a591e3 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
All reported replicas are:
A = d06d8f442c824758ae944ffe71e3cb44
B = 01cff8a283e94658aea5fad9fa0b2db4
C = 747982f4b834445b93f077dd1295b2ff
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | -1 | Yes
C | A B* C | 1 | -1 | Yes
Tablet 459e1d55c65e4857a7ccba6b13316243 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet b29f930721b943039d1df1f03e073a74 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | -1 | Yes
B | A B* C | 2 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:29:29.960134 8266 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:29:30.465812 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 106: this message will repeat every 5th retry.
I20260810 22:29:31.014742 8670 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20260810 22:29:31.014798 8486 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+--------------------+---------
e52d1e743df1434f88d46b1016be2ae7 | 127.8.18.190:40323 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.8.18.148:42749 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+--------------------+-------------+----------+----------------+-----------------
01cff8a283e94658aea5fad9fa0b2db4 | 127.8.18.130:33631 | HEALTHY | <none> | 3 | 0
747982f4b834445b93f077dd1295b2ff | 127.8.18.129:40911 | HEALTHY | <none> | 0 | 0
d06d8f442c824758ae944ffe71e3cb44 | 127.8.18.131:36507 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.8.18.131:36507: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+----------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.8.18.130:33631, 127.8.18.129:40911
ipki_server_key_size | 768 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.129 | experimental | 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.130 | experimental | 127.8.18.130:33631
never_fsync | true | unsafe,advanced | 127.8.18.130:33631, 127.8.18.129:40911
openssl_security_level_override | 0 | unsafe,hidden | 127.8.18.130:33631, 127.8.18.129:40911
rpc_server_allow_ephemeral_ports | true | unsafe | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_format | pb | hidden | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-0/data/info.pb | hidden | 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-1/data/info.pb | hidden | 127.8.18.130:33631
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+--------------------+----------------------------------------
builtin_ntp_servers | 127.8.18.148:42749 | 127.8.18.130:33631, 127.8.18.129:40911
time_source | builtin | 127.8.18.130:33631, 127.8.18.129:40911
Version Summary
Version | Servers
-----------------+-----------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.8.18.190:40323, tserver@127.8.18.129:40911, tserver@127.8.18.130:33631
Tablet Summary
Tablet 1f99182d93ed483e9f520bf312a591e3 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
All reported replicas are:
A = d06d8f442c824758ae944ffe71e3cb44
B = 01cff8a283e94658aea5fad9fa0b2db4
C = 747982f4b834445b93f077dd1295b2ff
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | -1 | Yes
C | A B* C | 1 | -1 | Yes
Tablet 459e1d55c65e4857a7ccba6b13316243 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet b29f930721b943039d1df1f03e073a74 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | -1 | Yes
B | A B* C | 2 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:29:31.039361 8266 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:29:31.480634 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 106: this message will repeat every 5th retry.
I20260810 22:29:32.113132 8670 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20260810 22:29:32.117971 8486 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+--------------------+---------
e52d1e743df1434f88d46b1016be2ae7 | 127.8.18.190:40323 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.8.18.148:42749 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+--------------------+-------------+----------+----------------+-----------------
01cff8a283e94658aea5fad9fa0b2db4 | 127.8.18.130:33631 | HEALTHY | <none> | 3 | 0
747982f4b834445b93f077dd1295b2ff | 127.8.18.129:40911 | HEALTHY | <none> | 0 | 0
d06d8f442c824758ae944ffe71e3cb44 | 127.8.18.131:36507 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.8.18.131:36507: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+----------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.8.18.130:33631, 127.8.18.129:40911
ipki_server_key_size | 768 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.129 | experimental | 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.130 | experimental | 127.8.18.130:33631
never_fsync | true | unsafe,advanced | 127.8.18.130:33631, 127.8.18.129:40911
openssl_security_level_override | 0 | unsafe,hidden | 127.8.18.130:33631, 127.8.18.129:40911
rpc_server_allow_ephemeral_ports | true | unsafe | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_format | pb | hidden | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-0/data/info.pb | hidden | 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-1/data/info.pb | hidden | 127.8.18.130:33631
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+--------------------+----------------------------------------
builtin_ntp_servers | 127.8.18.148:42749 | 127.8.18.130:33631, 127.8.18.129:40911
time_source | builtin | 127.8.18.130:33631, 127.8.18.129:40911
Version Summary
Version | Servers
-----------------+-----------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.8.18.190:40323, tserver@127.8.18.130:33631, tserver@127.8.18.129:40911
Tablet Summary
Tablet 1f99182d93ed483e9f520bf312a591e3 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
All reported replicas are:
A = d06d8f442c824758ae944ffe71e3cb44
B = 01cff8a283e94658aea5fad9fa0b2db4
C = 747982f4b834445b93f077dd1295b2ff
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | -1 | Yes
C | A B* C | 1 | -1 | Yes
Tablet 459e1d55c65e4857a7ccba6b13316243 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet b29f930721b943039d1df1f03e073a74 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | -1 | Yes
B | A B* C | 2 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:29:32.149837 8266 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:29:32.420258 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 106: this message will repeat every 5th retry.
W20260810 22:29:32.825266 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 111: this message will repeat every 5th retry.
I20260810 22:29:33.202704 8670 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20260810 22:29:33.203394 8486 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+--------------------+---------
e52d1e743df1434f88d46b1016be2ae7 | 127.8.18.190:40323 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.8.18.148:42749 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+--------------------+-------------+----------+----------------+-----------------
01cff8a283e94658aea5fad9fa0b2db4 | 127.8.18.130:33631 | HEALTHY | <none> | 3 | 0
747982f4b834445b93f077dd1295b2ff | 127.8.18.129:40911 | HEALTHY | <none> | 0 | 0
d06d8f442c824758ae944ffe71e3cb44 | 127.8.18.131:36507 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.8.18.131:36507: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+----------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.8.18.130:33631, 127.8.18.129:40911
ipki_server_key_size | 768 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.129 | experimental | 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.130 | experimental | 127.8.18.130:33631
never_fsync | true | unsafe,advanced | 127.8.18.130:33631, 127.8.18.129:40911
openssl_security_level_override | 0 | unsafe,hidden | 127.8.18.130:33631, 127.8.18.129:40911
rpc_server_allow_ephemeral_ports | true | unsafe | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_format | pb | hidden | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-0/data/info.pb | hidden | 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-1/data/info.pb | hidden | 127.8.18.130:33631
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+--------------------+----------------------------------------
builtin_ntp_servers | 127.8.18.148:42749 | 127.8.18.130:33631, 127.8.18.129:40911
time_source | builtin | 127.8.18.130:33631, 127.8.18.129:40911
Version Summary
Version | Servers
-----------------+-----------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.8.18.190:40323, tserver@127.8.18.130:33631, tserver@127.8.18.129:40911
Tablet Summary
Tablet 1f99182d93ed483e9f520bf312a591e3 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
All reported replicas are:
A = d06d8f442c824758ae944ffe71e3cb44
B = 01cff8a283e94658aea5fad9fa0b2db4
C = 747982f4b834445b93f077dd1295b2ff
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | -1 | Yes
C | A B* C | 1 | -1 | Yes
Tablet 459e1d55c65e4857a7ccba6b13316243 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet b29f930721b943039d1df1f03e073a74 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | -1 | Yes
B | A B* C | 2 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:29:33.223225 8266 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:29:34.018548 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 111: this message will repeat every 5th retry.
I20260810 22:29:34.286576 8486 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20260810 22:29:34.286576 8670 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+--------------------+---------
e52d1e743df1434f88d46b1016be2ae7 | 127.8.18.190:40323 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.8.18.148:42749 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+--------------------+-------------+----------+----------------+-----------------
01cff8a283e94658aea5fad9fa0b2db4 | 127.8.18.130:33631 | HEALTHY | <none> | 3 | 0
747982f4b834445b93f077dd1295b2ff | 127.8.18.129:40911 | HEALTHY | <none> | 0 | 0
d06d8f442c824758ae944ffe71e3cb44 | 127.8.18.131:36507 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.8.18.131:36507: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+----------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.8.18.130:33631, 127.8.18.129:40911
ipki_server_key_size | 768 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.129 | experimental | 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.130 | experimental | 127.8.18.130:33631
never_fsync | true | unsafe,advanced | 127.8.18.130:33631, 127.8.18.129:40911
openssl_security_level_override | 0 | unsafe,hidden | 127.8.18.130:33631, 127.8.18.129:40911
rpc_server_allow_ephemeral_ports | true | unsafe | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_format | pb | hidden | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-0/data/info.pb | hidden | 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-1/data/info.pb | hidden | 127.8.18.130:33631
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+--------------------+----------------------------------------
builtin_ntp_servers | 127.8.18.148:42749 | 127.8.18.130:33631, 127.8.18.129:40911
time_source | builtin | 127.8.18.130:33631, 127.8.18.129:40911
Version Summary
Version | Servers
-----------------+-----------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.8.18.190:40323, tserver@127.8.18.129:40911, tserver@127.8.18.130:33631
Tablet Summary
Tablet 1f99182d93ed483e9f520bf312a591e3 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
All reported replicas are:
A = d06d8f442c824758ae944ffe71e3cb44
B = 01cff8a283e94658aea5fad9fa0b2db4
C = 747982f4b834445b93f077dd1295b2ff
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | -1 | Yes
C | A B* C | 1 | -1 | Yes
Tablet 459e1d55c65e4857a7ccba6b13316243 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet b29f930721b943039d1df1f03e073a74 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | -1 | Yes
B | A B* C | 2 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:29:34.306469 8266 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:29:34.756531 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 30 similar messages]
W20260810 22:29:34.883725 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 111: this message will repeat every 5th retry.
W20260810 22:29:35.170374 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 116: this message will repeat every 5th retry.
I20260810 22:29:35.363487 8670 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20260810 22:29:35.364099 8486 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+--------------------+---------
e52d1e743df1434f88d46b1016be2ae7 | 127.8.18.190:40323 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.8.18.148:42749 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+--------------------+-------------+----------+----------------+-----------------
01cff8a283e94658aea5fad9fa0b2db4 | 127.8.18.130:33631 | HEALTHY | <none> | 3 | 0
747982f4b834445b93f077dd1295b2ff | 127.8.18.129:40911 | HEALTHY | <none> | 0 | 0
d06d8f442c824758ae944ffe71e3cb44 | 127.8.18.131:36507 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.8.18.131:36507: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+----------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.8.18.130:33631, 127.8.18.129:40911
ipki_server_key_size | 768 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.129 | experimental | 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.130 | experimental | 127.8.18.130:33631
never_fsync | true | unsafe,advanced | 127.8.18.130:33631, 127.8.18.129:40911
openssl_security_level_override | 0 | unsafe,hidden | 127.8.18.130:33631, 127.8.18.129:40911
rpc_server_allow_ephemeral_ports | true | unsafe | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_format | pb | hidden | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-0/data/info.pb | hidden | 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-1/data/info.pb | hidden | 127.8.18.130:33631
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+--------------------+----------------------------------------
builtin_ntp_servers | 127.8.18.148:42749 | 127.8.18.130:33631, 127.8.18.129:40911
time_source | builtin | 127.8.18.130:33631, 127.8.18.129:40911
Version Summary
Version | Servers
-----------------+-----------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.8.18.190:40323, tserver@127.8.18.130:33631, tserver@127.8.18.129:40911
Tablet Summary
Tablet 1f99182d93ed483e9f520bf312a591e3 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
All reported replicas are:
A = d06d8f442c824758ae944ffe71e3cb44
B = 01cff8a283e94658aea5fad9fa0b2db4
C = 747982f4b834445b93f077dd1295b2ff
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | -1 | Yes
C | A B* C | 1 | -1 | Yes
Tablet 459e1d55c65e4857a7ccba6b13316243 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet b29f930721b943039d1df1f03e073a74 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | -1 | Yes
B | A B* C | 2 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:29:35.386379 8266 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
I20260810 22:29:36.438755 8486 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20260810 22:29:36.439046 8670 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+--------------------+---------
e52d1e743df1434f88d46b1016be2ae7 | 127.8.18.190:40323 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.8.18.148:42749 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+--------------------+-------------+----------+----------------+-----------------
01cff8a283e94658aea5fad9fa0b2db4 | 127.8.18.130:33631 | HEALTHY | <none> | 3 | 0
747982f4b834445b93f077dd1295b2ff | 127.8.18.129:40911 | HEALTHY | <none> | 0 | 0
d06d8f442c824758ae944ffe71e3cb44 | 127.8.18.131:36507 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.8.18.131:36507: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+----------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.8.18.130:33631, 127.8.18.129:40911
ipki_server_key_size | 768 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.129 | experimental | 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.130 | experimental | 127.8.18.130:33631
never_fsync | true | unsafe,advanced | 127.8.18.130:33631, 127.8.18.129:40911
openssl_security_level_override | 0 | unsafe,hidden | 127.8.18.130:33631, 127.8.18.129:40911
rpc_server_allow_ephemeral_ports | true | unsafe | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_format | pb | hidden | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-0/data/info.pb | hidden | 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-1/data/info.pb | hidden | 127.8.18.130:33631
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+--------------------+----------------------------------------
builtin_ntp_servers | 127.8.18.148:42749 | 127.8.18.130:33631, 127.8.18.129:40911
time_source | builtin | 127.8.18.130:33631, 127.8.18.129:40911
Version Summary
Version | Servers
-----------------+-----------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.8.18.190:40323, tserver@127.8.18.129:40911, tserver@127.8.18.130:33631
Tablet Summary
Tablet 1f99182d93ed483e9f520bf312a591e3 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
All reported replicas are:
A = d06d8f442c824758ae944ffe71e3cb44
B = 01cff8a283e94658aea5fad9fa0b2db4
C = 747982f4b834445b93f077dd1295b2ff
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | -1 | Yes
C | A B* C | 1 | -1 | Yes
Tablet 459e1d55c65e4857a7ccba6b13316243 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet b29f930721b943039d1df1f03e073a74 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | -1 | Yes
B | A B* C | 2 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:29:36.457545 8266 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:29:36.602483 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 116: this message will repeat every 5th retry.
W20260810 22:29:37.235585 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 116: this message will repeat every 5th retry.
I20260810 22:29:37.511772 8486 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20260810 22:29:37.511822 8670 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+--------------------+---------
e52d1e743df1434f88d46b1016be2ae7 | 127.8.18.190:40323 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.8.18.148:42749 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+--------------------+-------------+----------+----------------+-----------------
01cff8a283e94658aea5fad9fa0b2db4 | 127.8.18.130:33631 | HEALTHY | <none> | 3 | 0
747982f4b834445b93f077dd1295b2ff | 127.8.18.129:40911 | HEALTHY | <none> | 0 | 0
d06d8f442c824758ae944ffe71e3cb44 | 127.8.18.131:36507 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.8.18.131:36507: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+----------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.8.18.130:33631, 127.8.18.129:40911
ipki_server_key_size | 768 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.129 | experimental | 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.130 | experimental | 127.8.18.130:33631
never_fsync | true | unsafe,advanced | 127.8.18.130:33631, 127.8.18.129:40911
openssl_security_level_override | 0 | unsafe,hidden | 127.8.18.130:33631, 127.8.18.129:40911
rpc_server_allow_ephemeral_ports | true | unsafe | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_format | pb | hidden | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-0/data/info.pb | hidden | 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-1/data/info.pb | hidden | 127.8.18.130:33631
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+--------------------+----------------------------------------
builtin_ntp_servers | 127.8.18.148:42749 | 127.8.18.130:33631, 127.8.18.129:40911
time_source | builtin | 127.8.18.130:33631, 127.8.18.129:40911
Version Summary
Version | Servers
-----------------+-----------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.8.18.190:40323, tserver@127.8.18.129:40911, tserver@127.8.18.130:33631
Tablet Summary
Tablet 1f99182d93ed483e9f520bf312a591e3 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
All reported replicas are:
A = d06d8f442c824758ae944ffe71e3cb44
B = 01cff8a283e94658aea5fad9fa0b2db4
C = 747982f4b834445b93f077dd1295b2ff
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | -1 | Yes
C | A B* C | 1 | -1 | Yes
Tablet 459e1d55c65e4857a7ccba6b13316243 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet b29f930721b943039d1df1f03e073a74 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | -1 | Yes
B | A B* C | 2 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:29:37.534631 8266 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:29:37.588068 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 121: this message will repeat every 5th retry.
I20260810 22:29:38.587431 8486 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20260810 22:29:38.587520 8670 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+--------------------+---------
e52d1e743df1434f88d46b1016be2ae7 | 127.8.18.190:40323 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.8.18.148:42749 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+--------------------+-------------+----------+----------------+-----------------
01cff8a283e94658aea5fad9fa0b2db4 | 127.8.18.130:33631 | HEALTHY | <none> | 3 | 0
747982f4b834445b93f077dd1295b2ff | 127.8.18.129:40911 | HEALTHY | <none> | 0 | 0
d06d8f442c824758ae944ffe71e3cb44 | 127.8.18.131:36507 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.8.18.131:36507: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+----------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.8.18.130:33631, 127.8.18.129:40911
ipki_server_key_size | 768 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.129 | experimental | 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.130 | experimental | 127.8.18.130:33631
never_fsync | true | unsafe,advanced | 127.8.18.130:33631, 127.8.18.129:40911
openssl_security_level_override | 0 | unsafe,hidden | 127.8.18.130:33631, 127.8.18.129:40911
rpc_server_allow_ephemeral_ports | true | unsafe | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_format | pb | hidden | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-0/data/info.pb | hidden | 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-1/data/info.pb | hidden | 127.8.18.130:33631
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+--------------------+----------------------------------------
builtin_ntp_servers | 127.8.18.148:42749 | 127.8.18.130:33631, 127.8.18.129:40911
time_source | builtin | 127.8.18.130:33631, 127.8.18.129:40911
Version Summary
Version | Servers
-----------------+-----------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.8.18.190:40323, tserver@127.8.18.129:40911, tserver@127.8.18.130:33631
Tablet Summary
Tablet 1f99182d93ed483e9f520bf312a591e3 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
All reported replicas are:
A = d06d8f442c824758ae944ffe71e3cb44
B = 01cff8a283e94658aea5fad9fa0b2db4
C = 747982f4b834445b93f077dd1295b2ff
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | -1 | Yes
C | A B* C | 1 | -1 | Yes
Tablet 459e1d55c65e4857a7ccba6b13316243 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet b29f930721b943039d1df1f03e073a74 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | -1 | Yes
B | A B* C | 2 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:29:38.607560 8266 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:29:38.899737 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 121: this message will repeat every 5th retry.
W20260810 22:29:39.608693 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 121: this message will repeat every 5th retry.
I20260810 22:29:39.665445 8670 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20260810 22:29:39.665848 8486 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+--------------------+---------
e52d1e743df1434f88d46b1016be2ae7 | 127.8.18.190:40323 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.8.18.148:42749 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+--------------------+-------------+----------+----------------+-----------------
01cff8a283e94658aea5fad9fa0b2db4 | 127.8.18.130:33631 | HEALTHY | <none> | 3 | 0
747982f4b834445b93f077dd1295b2ff | 127.8.18.129:40911 | HEALTHY | <none> | 0 | 0
d06d8f442c824758ae944ffe71e3cb44 | 127.8.18.131:36507 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.8.18.131:36507: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+----------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.8.18.130:33631, 127.8.18.129:40911
ipki_server_key_size | 768 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.129 | experimental | 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.130 | experimental | 127.8.18.130:33631
never_fsync | true | unsafe,advanced | 127.8.18.130:33631, 127.8.18.129:40911
openssl_security_level_override | 0 | unsafe,hidden | 127.8.18.130:33631, 127.8.18.129:40911
rpc_server_allow_ephemeral_ports | true | unsafe | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_format | pb | hidden | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-0/data/info.pb | hidden | 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-1/data/info.pb | hidden | 127.8.18.130:33631
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+--------------------+----------------------------------------
builtin_ntp_servers | 127.8.18.148:42749 | 127.8.18.130:33631, 127.8.18.129:40911
time_source | builtin | 127.8.18.130:33631, 127.8.18.129:40911
Version Summary
Version | Servers
-----------------+-----------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.8.18.190:40323, tserver@127.8.18.130:33631, tserver@127.8.18.129:40911
Tablet Summary
Tablet 1f99182d93ed483e9f520bf312a591e3 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
All reported replicas are:
A = d06d8f442c824758ae944ffe71e3cb44
B = 01cff8a283e94658aea5fad9fa0b2db4
C = 747982f4b834445b93f077dd1295b2ff
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | -1 | Yes
C | A B* C | 1 | -1 | Yes
Tablet 459e1d55c65e4857a7ccba6b13316243 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet b29f930721b943039d1df1f03e073a74 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | -1 | Yes
B | A B* C | 2 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:29:39.687310 8266 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:29:39.777613 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 30 similar messages]
W20260810 22:29:40.417701 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 126: this message will repeat every 5th retry.
I20260810 22:29:40.756958 8670 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20260810 22:29:40.760216 8486 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+--------------------+---------
e52d1e743df1434f88d46b1016be2ae7 | 127.8.18.190:40323 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.8.18.148:42749 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+--------------------+-------------+----------+----------------+-----------------
01cff8a283e94658aea5fad9fa0b2db4 | 127.8.18.130:33631 | HEALTHY | <none> | 3 | 0
747982f4b834445b93f077dd1295b2ff | 127.8.18.129:40911 | HEALTHY | <none> | 0 | 0
d06d8f442c824758ae944ffe71e3cb44 | 127.8.18.131:36507 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.8.18.131:36507: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+----------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.8.18.130:33631, 127.8.18.129:40911
ipki_server_key_size | 768 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.129 | experimental | 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.130 | experimental | 127.8.18.130:33631
never_fsync | true | unsafe,advanced | 127.8.18.130:33631, 127.8.18.129:40911
openssl_security_level_override | 0 | unsafe,hidden | 127.8.18.130:33631, 127.8.18.129:40911
rpc_server_allow_ephemeral_ports | true | unsafe | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_format | pb | hidden | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-0/data/info.pb | hidden | 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-1/data/info.pb | hidden | 127.8.18.130:33631
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+--------------------+----------------------------------------
builtin_ntp_servers | 127.8.18.148:42749 | 127.8.18.130:33631, 127.8.18.129:40911
time_source | builtin | 127.8.18.130:33631, 127.8.18.129:40911
Version Summary
Version | Servers
-----------------+-----------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.8.18.190:40323, tserver@127.8.18.130:33631, tserver@127.8.18.129:40911
Tablet Summary
Tablet 1f99182d93ed483e9f520bf312a591e3 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
All reported replicas are:
A = d06d8f442c824758ae944ffe71e3cb44
B = 01cff8a283e94658aea5fad9fa0b2db4
C = 747982f4b834445b93f077dd1295b2ff
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | -1 | Yes
C | A B* C | 1 | -1 | Yes
Tablet 459e1d55c65e4857a7ccba6b13316243 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet b29f930721b943039d1df1f03e073a74 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | -1 | Yes
B | A B* C | 2 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:29:40.788939 8266 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:29:41.359083 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 126: this message will repeat every 5th retry.
I20260810 22:29:41.842724 8670 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20260810 22:29:41.842823 8486 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+--------------------+---------
e52d1e743df1434f88d46b1016be2ae7 | 127.8.18.190:40323 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.8.18.148:42749 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+--------------------+-------------+----------+----------------+-----------------
01cff8a283e94658aea5fad9fa0b2db4 | 127.8.18.130:33631 | HEALTHY | <none> | 3 | 0
747982f4b834445b93f077dd1295b2ff | 127.8.18.129:40911 | HEALTHY | <none> | 0 | 0
d06d8f442c824758ae944ffe71e3cb44 | 127.8.18.131:36507 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.8.18.131:36507: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+----------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.8.18.130:33631, 127.8.18.129:40911
ipki_server_key_size | 768 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.129 | experimental | 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.130 | experimental | 127.8.18.130:33631
never_fsync | true | unsafe,advanced | 127.8.18.130:33631, 127.8.18.129:40911
openssl_security_level_override | 0 | unsafe,hidden | 127.8.18.130:33631, 127.8.18.129:40911
rpc_server_allow_ephemeral_ports | true | unsafe | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_format | pb | hidden | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-0/data/info.pb | hidden | 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-1/data/info.pb | hidden | 127.8.18.130:33631
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+--------------------+----------------------------------------
builtin_ntp_servers | 127.8.18.148:42749 | 127.8.18.130:33631, 127.8.18.129:40911
time_source | builtin | 127.8.18.130:33631, 127.8.18.129:40911
Version Summary
Version | Servers
-----------------+-----------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.8.18.190:40323, tserver@127.8.18.129:40911, tserver@127.8.18.130:33631
Tablet Summary
Tablet 1f99182d93ed483e9f520bf312a591e3 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
All reported replicas are:
A = d06d8f442c824758ae944ffe71e3cb44
B = 01cff8a283e94658aea5fad9fa0b2db4
C = 747982f4b834445b93f077dd1295b2ff
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | -1 | Yes
C | A B* C | 1 | -1 | Yes
Tablet 459e1d55c65e4857a7ccba6b13316243 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet b29f930721b943039d1df1f03e073a74 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | -1 | Yes
B | A B* C | 2 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:29:41.867974 8266 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:29:42.165751 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 126: this message will repeat every 5th retry.
I20260810 22:29:42.921728 8670 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20260810 22:29:42.921772 8486 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+--------------------+---------
e52d1e743df1434f88d46b1016be2ae7 | 127.8.18.190:40323 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.8.18.148:42749 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+--------------------+-------------+----------+----------------+-----------------
01cff8a283e94658aea5fad9fa0b2db4 | 127.8.18.130:33631 | HEALTHY | <none> | 3 | 0
747982f4b834445b93f077dd1295b2ff | 127.8.18.129:40911 | HEALTHY | <none> | 0 | 0
d06d8f442c824758ae944ffe71e3cb44 | 127.8.18.131:36507 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.8.18.131:36507: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+----------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.8.18.130:33631, 127.8.18.129:40911
ipki_server_key_size | 768 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.129 | experimental | 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.130 | experimental | 127.8.18.130:33631
never_fsync | true | unsafe,advanced | 127.8.18.130:33631, 127.8.18.129:40911
openssl_security_level_override | 0 | unsafe,hidden | 127.8.18.130:33631, 127.8.18.129:40911
rpc_server_allow_ephemeral_ports | true | unsafe | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_format | pb | hidden | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-0/data/info.pb | hidden | 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-1/data/info.pb | hidden | 127.8.18.130:33631
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+--------------------+----------------------------------------
builtin_ntp_servers | 127.8.18.148:42749 | 127.8.18.130:33631, 127.8.18.129:40911
time_source | builtin | 127.8.18.130:33631, 127.8.18.129:40911
Version Summary
Version | Servers
-----------------+-----------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.8.18.190:40323, tserver@127.8.18.129:40911, tserver@127.8.18.130:33631
Tablet Summary
Tablet 1f99182d93ed483e9f520bf312a591e3 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
All reported replicas are:
A = d06d8f442c824758ae944ffe71e3cb44
B = 01cff8a283e94658aea5fad9fa0b2db4
C = 747982f4b834445b93f077dd1295b2ff
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | -1 | Yes
C | A B* C | 1 | -1 | Yes
Tablet 459e1d55c65e4857a7ccba6b13316243 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet b29f930721b943039d1df1f03e073a74 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | -1 | Yes
B | A B* C | 2 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:29:42.941025 8266 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:29:43.094969 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 131: this message will repeat every 5th retry.
W20260810 22:29:43.939836 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 131: this message will repeat every 5th retry.
I20260810 22:29:43.997869 8670 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20260810 22:29:43.997866 8486 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+--------------------+---------
e52d1e743df1434f88d46b1016be2ae7 | 127.8.18.190:40323 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.8.18.148:42749 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+--------------------+-------------+----------+----------------+-----------------
01cff8a283e94658aea5fad9fa0b2db4 | 127.8.18.130:33631 | HEALTHY | <none> | 3 | 0
747982f4b834445b93f077dd1295b2ff | 127.8.18.129:40911 | HEALTHY | <none> | 0 | 0
d06d8f442c824758ae944ffe71e3cb44 | 127.8.18.131:36507 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.8.18.131:36507: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+----------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.8.18.130:33631, 127.8.18.129:40911
ipki_server_key_size | 768 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.129 | experimental | 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.130 | experimental | 127.8.18.130:33631
never_fsync | true | unsafe,advanced | 127.8.18.130:33631, 127.8.18.129:40911
openssl_security_level_override | 0 | unsafe,hidden | 127.8.18.130:33631, 127.8.18.129:40911
rpc_server_allow_ephemeral_ports | true | unsafe | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_format | pb | hidden | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-0/data/info.pb | hidden | 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-1/data/info.pb | hidden | 127.8.18.130:33631
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+--------------------+----------------------------------------
builtin_ntp_servers | 127.8.18.148:42749 | 127.8.18.130:33631, 127.8.18.129:40911
time_source | builtin | 127.8.18.130:33631, 127.8.18.129:40911
Version Summary
Version | Servers
-----------------+-----------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.8.18.190:40323, tserver@127.8.18.129:40911, tserver@127.8.18.130:33631
Tablet Summary
Tablet 1f99182d93ed483e9f520bf312a591e3 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
All reported replicas are:
A = d06d8f442c824758ae944ffe71e3cb44
B = 01cff8a283e94658aea5fad9fa0b2db4
C = 747982f4b834445b93f077dd1295b2ff
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | -1 | Yes
C | A B* C | 1 | -1 | Yes
Tablet 459e1d55c65e4857a7ccba6b13316243 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet b29f930721b943039d1df1f03e073a74 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | -1 | Yes
B | A B* C | 2 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:29:44.018255 8266 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:29:44.776644 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 131: this message will repeat every 5th retry.
W20260810 22:29:45.057152 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 29 similar messages]
I20260810 22:29:45.079746 8670 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20260810 22:29:45.080139 8486 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+--------------------+---------
e52d1e743df1434f88d46b1016be2ae7 | 127.8.18.190:40323 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.8.18.148:42749 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+--------------------+-------------+----------+----------------+-----------------
01cff8a283e94658aea5fad9fa0b2db4 | 127.8.18.130:33631 | HEALTHY | <none> | 3 | 0
747982f4b834445b93f077dd1295b2ff | 127.8.18.129:40911 | HEALTHY | <none> | 0 | 0
d06d8f442c824758ae944ffe71e3cb44 | 127.8.18.131:36507 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.8.18.131:36507: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+----------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.8.18.130:33631, 127.8.18.129:40911
ipki_server_key_size | 768 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.129 | experimental | 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.130 | experimental | 127.8.18.130:33631
never_fsync | true | unsafe,advanced | 127.8.18.130:33631, 127.8.18.129:40911
openssl_security_level_override | 0 | unsafe,hidden | 127.8.18.130:33631, 127.8.18.129:40911
rpc_server_allow_ephemeral_ports | true | unsafe | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_format | pb | hidden | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-0/data/info.pb | hidden | 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-1/data/info.pb | hidden | 127.8.18.130:33631
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+--------------------+----------------------------------------
builtin_ntp_servers | 127.8.18.148:42749 | 127.8.18.130:33631, 127.8.18.129:40911
time_source | builtin | 127.8.18.130:33631, 127.8.18.129:40911
Version Summary
Version | Servers
-----------------+-----------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.8.18.190:40323, tserver@127.8.18.130:33631, tserver@127.8.18.129:40911
Tablet Summary
Tablet 1f99182d93ed483e9f520bf312a591e3 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
All reported replicas are:
A = d06d8f442c824758ae944ffe71e3cb44
B = 01cff8a283e94658aea5fad9fa0b2db4
C = 747982f4b834445b93f077dd1295b2ff
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | -1 | Yes
C | A B* C | 1 | -1 | Yes
Tablet 459e1d55c65e4857a7ccba6b13316243 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet b29f930721b943039d1df1f03e073a74 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | -1 | Yes
B | A B* C | 2 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:29:45.104625 8266 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:29:45.511768 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 136: this message will repeat every 5th retry.
I20260810 22:29:46.159842 8486 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20260810 22:29:46.159912 8670 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+--------------------+---------
e52d1e743df1434f88d46b1016be2ae7 | 127.8.18.190:40323 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.8.18.148:42749 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+--------------------+-------------+----------+----------------+-----------------
01cff8a283e94658aea5fad9fa0b2db4 | 127.8.18.130:33631 | HEALTHY | <none> | 3 | 0
747982f4b834445b93f077dd1295b2ff | 127.8.18.129:40911 | HEALTHY | <none> | 0 | 0
d06d8f442c824758ae944ffe71e3cb44 | 127.8.18.131:36507 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.8.18.131:36507: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+----------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.8.18.130:33631, 127.8.18.129:40911
ipki_server_key_size | 768 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.129 | experimental | 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.130 | experimental | 127.8.18.130:33631
never_fsync | true | unsafe,advanced | 127.8.18.130:33631, 127.8.18.129:40911
openssl_security_level_override | 0 | unsafe,hidden | 127.8.18.130:33631, 127.8.18.129:40911
rpc_server_allow_ephemeral_ports | true | unsafe | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_format | pb | hidden | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-0/data/info.pb | hidden | 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-1/data/info.pb | hidden | 127.8.18.130:33631
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+--------------------+----------------------------------------
builtin_ntp_servers | 127.8.18.148:42749 | 127.8.18.130:33631, 127.8.18.129:40911
time_source | builtin | 127.8.18.130:33631, 127.8.18.129:40911
Version Summary
Version | Servers
-----------------+-----------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.8.18.190:40323, tserver@127.8.18.129:40911, tserver@127.8.18.130:33631
Tablet Summary
Tablet 1f99182d93ed483e9f520bf312a591e3 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
All reported replicas are:
A = d06d8f442c824758ae944ffe71e3cb44
B = 01cff8a283e94658aea5fad9fa0b2db4
C = 747982f4b834445b93f077dd1295b2ff
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | -1 | Yes
C | A B* C | 1 | -1 | Yes
Tablet 459e1d55c65e4857a7ccba6b13316243 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet b29f930721b943039d1df1f03e073a74 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | -1 | Yes
B | A B* C | 2 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:29:46.179317 8266 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:29:46.628176 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 136: this message will repeat every 5th retry.
I20260810 22:29:47.233831 8670 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20260810 22:29:47.234054 8486 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+--------------------+---------
e52d1e743df1434f88d46b1016be2ae7 | 127.8.18.190:40323 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.8.18.148:42749 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+--------------------+-------------+----------+----------------+-----------------
01cff8a283e94658aea5fad9fa0b2db4 | 127.8.18.130:33631 | HEALTHY | <none> | 3 | 0
747982f4b834445b93f077dd1295b2ff | 127.8.18.129:40911 | HEALTHY | <none> | 0 | 0
d06d8f442c824758ae944ffe71e3cb44 | 127.8.18.131:36507 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.8.18.131:36507: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+----------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.8.18.130:33631, 127.8.18.129:40911
ipki_server_key_size | 768 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.129 | experimental | 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.130 | experimental | 127.8.18.130:33631
never_fsync | true | unsafe,advanced | 127.8.18.130:33631, 127.8.18.129:40911
openssl_security_level_override | 0 | unsafe,hidden | 127.8.18.130:33631, 127.8.18.129:40911
rpc_server_allow_ephemeral_ports | true | unsafe | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_format | pb | hidden | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-0/data/info.pb | hidden | 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-1/data/info.pb | hidden | 127.8.18.130:33631
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+--------------------+----------------------------------------
builtin_ntp_servers | 127.8.18.148:42749 | 127.8.18.130:33631, 127.8.18.129:40911
time_source | builtin | 127.8.18.130:33631, 127.8.18.129:40911
Version Summary
Version | Servers
-----------------+-----------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.8.18.190:40323, tserver@127.8.18.130:33631, tserver@127.8.18.129:40911
Tablet Summary
Tablet 1f99182d93ed483e9f520bf312a591e3 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
All reported replicas are:
A = d06d8f442c824758ae944ffe71e3cb44
B = 01cff8a283e94658aea5fad9fa0b2db4
C = 747982f4b834445b93f077dd1295b2ff
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | -1 | Yes
C | A B* C | 1 | -1 | Yes
Tablet 459e1d55c65e4857a7ccba6b13316243 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet b29f930721b943039d1df1f03e073a74 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | -1 | Yes
B | A B* C | 2 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:29:47.254369 8266 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:29:47.520303 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 136: this message will repeat every 5th retry.
W20260810 22:29:48.123590 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 141: this message will repeat every 5th retry.
I20260810 22:29:48.309870 8670 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20260810 22:29:48.310446 8486 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+--------------------+---------
e52d1e743df1434f88d46b1016be2ae7 | 127.8.18.190:40323 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.8.18.148:42749 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+--------------------+-------------+----------+----------------+-----------------
01cff8a283e94658aea5fad9fa0b2db4 | 127.8.18.130:33631 | HEALTHY | <none> | 3 | 0
747982f4b834445b93f077dd1295b2ff | 127.8.18.129:40911 | HEALTHY | <none> | 0 | 0
d06d8f442c824758ae944ffe71e3cb44 | 127.8.18.131:36507 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.8.18.131:36507: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+----------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.8.18.130:33631, 127.8.18.129:40911
ipki_server_key_size | 768 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.129 | experimental | 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.130 | experimental | 127.8.18.130:33631
never_fsync | true | unsafe,advanced | 127.8.18.130:33631, 127.8.18.129:40911
openssl_security_level_override | 0 | unsafe,hidden | 127.8.18.130:33631, 127.8.18.129:40911
rpc_server_allow_ephemeral_ports | true | unsafe | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_format | pb | hidden | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-0/data/info.pb | hidden | 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-1/data/info.pb | hidden | 127.8.18.130:33631
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+--------------------+----------------------------------------
builtin_ntp_servers | 127.8.18.148:42749 | 127.8.18.130:33631, 127.8.18.129:40911
time_source | builtin | 127.8.18.130:33631, 127.8.18.129:40911
Version Summary
Version | Servers
-----------------+-----------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.8.18.190:40323, tserver@127.8.18.130:33631, tserver@127.8.18.129:40911
Tablet Summary
Tablet 1f99182d93ed483e9f520bf312a591e3 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
All reported replicas are:
A = d06d8f442c824758ae944ffe71e3cb44
B = 01cff8a283e94658aea5fad9fa0b2db4
C = 747982f4b834445b93f077dd1295b2ff
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | -1 | Yes
C | A B* C | 1 | -1 | Yes
Tablet 459e1d55c65e4857a7ccba6b13316243 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet b29f930721b943039d1df1f03e073a74 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | -1 | Yes
B | A B* C | 2 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:29:48.333724 8266 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:29:49.299763 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 141: this message will repeat every 5th retry.
I20260810 22:29:49.387650 8486 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20260810 22:29:49.387900 8670 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+--------------------+---------
e52d1e743df1434f88d46b1016be2ae7 | 127.8.18.190:40323 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.8.18.148:42749 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+--------------------+-------------+----------+----------------+-----------------
01cff8a283e94658aea5fad9fa0b2db4 | 127.8.18.130:33631 | HEALTHY | <none> | 3 | 0
747982f4b834445b93f077dd1295b2ff | 127.8.18.129:40911 | HEALTHY | <none> | 0 | 0
d06d8f442c824758ae944ffe71e3cb44 | 127.8.18.131:36507 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.8.18.131:36507: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+----------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.8.18.130:33631, 127.8.18.129:40911
ipki_server_key_size | 768 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.129 | experimental | 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.130 | experimental | 127.8.18.130:33631
never_fsync | true | unsafe,advanced | 127.8.18.130:33631, 127.8.18.129:40911
openssl_security_level_override | 0 | unsafe,hidden | 127.8.18.130:33631, 127.8.18.129:40911
rpc_server_allow_ephemeral_ports | true | unsafe | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_format | pb | hidden | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-0/data/info.pb | hidden | 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-1/data/info.pb | hidden | 127.8.18.130:33631
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+--------------------+----------------------------------------
builtin_ntp_servers | 127.8.18.148:42749 | 127.8.18.130:33631, 127.8.18.129:40911
time_source | builtin | 127.8.18.130:33631, 127.8.18.129:40911
Version Summary
Version | Servers
-----------------+-----------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.8.18.190:40323, tserver@127.8.18.129:40911, tserver@127.8.18.130:33631
Tablet Summary
Tablet 1f99182d93ed483e9f520bf312a591e3 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
All reported replicas are:
A = d06d8f442c824758ae944ffe71e3cb44
B = 01cff8a283e94658aea5fad9fa0b2db4
C = 747982f4b834445b93f077dd1295b2ff
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | -1 | Yes
C | A B* C | 1 | -1 | Yes
Tablet 459e1d55c65e4857a7ccba6b13316243 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet b29f930721b943039d1df1f03e073a74 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | -1 | Yes
B | A B* C | 2 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:29:49.411713 8266 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:29:49.935639 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 141: this message will repeat every 5th retry.
W20260810 22:29:50.342029 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 30 similar messages]
W20260810 22:29:50.342777 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 146: this message will repeat every 5th retry.
I20260810 22:29:50.470098 8670 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20260810 22:29:50.470294 8486 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+--------------------+---------
e52d1e743df1434f88d46b1016be2ae7 | 127.8.18.190:40323 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.8.18.148:42749 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+--------------------+-------------+----------+----------------+-----------------
01cff8a283e94658aea5fad9fa0b2db4 | 127.8.18.130:33631 | HEALTHY | <none> | 3 | 0
747982f4b834445b93f077dd1295b2ff | 127.8.18.129:40911 | HEALTHY | <none> | 0 | 0
d06d8f442c824758ae944ffe71e3cb44 | 127.8.18.131:36507 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.8.18.131:36507: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+----------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.8.18.130:33631, 127.8.18.129:40911
ipki_server_key_size | 768 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.129 | experimental | 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.130 | experimental | 127.8.18.130:33631
never_fsync | true | unsafe,advanced | 127.8.18.130:33631, 127.8.18.129:40911
openssl_security_level_override | 0 | unsafe,hidden | 127.8.18.130:33631, 127.8.18.129:40911
rpc_server_allow_ephemeral_ports | true | unsafe | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_format | pb | hidden | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-0/data/info.pb | hidden | 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-1/data/info.pb | hidden | 127.8.18.130:33631
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+--------------------+----------------------------------------
builtin_ntp_servers | 127.8.18.148:42749 | 127.8.18.130:33631, 127.8.18.129:40911
time_source | builtin | 127.8.18.130:33631, 127.8.18.129:40911
Version Summary
Version | Servers
-----------------+-----------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.8.18.190:40323, tserver@127.8.18.130:33631, tserver@127.8.18.129:40911
Tablet Summary
Tablet 1f99182d93ed483e9f520bf312a591e3 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
All reported replicas are:
A = d06d8f442c824758ae944ffe71e3cb44
B = 01cff8a283e94658aea5fad9fa0b2db4
C = 747982f4b834445b93f077dd1295b2ff
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | -1 | Yes
C | A B* C | 1 | -1 | Yes
Tablet 459e1d55c65e4857a7ccba6b13316243 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet b29f930721b943039d1df1f03e073a74 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | -1 | Yes
B | A B* C | 2 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:29:50.494067 8266 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
I20260810 22:29:51.549551 8670 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20260810 22:29:51.550141 8486 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+--------------------+---------
e52d1e743df1434f88d46b1016be2ae7 | 127.8.18.190:40323 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.8.18.148:42749 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+--------------------+-------------+----------+----------------+-----------------
01cff8a283e94658aea5fad9fa0b2db4 | 127.8.18.130:33631 | HEALTHY | <none> | 3 | 0
747982f4b834445b93f077dd1295b2ff | 127.8.18.129:40911 | HEALTHY | <none> | 0 | 0
d06d8f442c824758ae944ffe71e3cb44 | 127.8.18.131:36507 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.8.18.131:36507: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+----------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.8.18.130:33631, 127.8.18.129:40911
ipki_server_key_size | 768 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.129 | experimental | 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.130 | experimental | 127.8.18.130:33631
never_fsync | true | unsafe,advanced | 127.8.18.130:33631, 127.8.18.129:40911
openssl_security_level_override | 0 | unsafe,hidden | 127.8.18.130:33631, 127.8.18.129:40911
rpc_server_allow_ephemeral_ports | true | unsafe | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_format | pb | hidden | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-0/data/info.pb | hidden | 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-1/data/info.pb | hidden | 127.8.18.130:33631
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+--------------------+----------------------------------------
builtin_ntp_servers | 127.8.18.148:42749 | 127.8.18.130:33631, 127.8.18.129:40911
time_source | builtin | 127.8.18.130:33631, 127.8.18.129:40911
Version Summary
Version | Servers
-----------------+-----------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.8.18.190:40323, tserver@127.8.18.130:33631, tserver@127.8.18.129:40911
Tablet Summary
Tablet 1f99182d93ed483e9f520bf312a591e3 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
All reported replicas are:
A = d06d8f442c824758ae944ffe71e3cb44
B = 01cff8a283e94658aea5fad9fa0b2db4
C = 747982f4b834445b93f077dd1295b2ff
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | -1 | Yes
C | A B* C | 1 | -1 | Yes
Tablet 459e1d55c65e4857a7ccba6b13316243 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet b29f930721b943039d1df1f03e073a74 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | -1 | Yes
B | A B* C | 2 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:29:51.572470 8266 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:29:51.859670 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 146: this message will repeat every 5th retry.
W20260810 22:29:52.425782 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 146: this message will repeat every 5th retry.
I20260810 22:29:52.625712 8670 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20260810 22:29:52.625962 8486 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+--------------------+---------
e52d1e743df1434f88d46b1016be2ae7 | 127.8.18.190:40323 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.8.18.148:42749 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+--------------------+-------------+----------+----------------+-----------------
01cff8a283e94658aea5fad9fa0b2db4 | 127.8.18.130:33631 | HEALTHY | <none> | 3 | 0
747982f4b834445b93f077dd1295b2ff | 127.8.18.129:40911 | HEALTHY | <none> | 0 | 0
d06d8f442c824758ae944ffe71e3cb44 | 127.8.18.131:36507 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.8.18.131:36507: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+----------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.8.18.130:33631, 127.8.18.129:40911
ipki_server_key_size | 768 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.129 | experimental | 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.130 | experimental | 127.8.18.130:33631
never_fsync | true | unsafe,advanced | 127.8.18.130:33631, 127.8.18.129:40911
openssl_security_level_override | 0 | unsafe,hidden | 127.8.18.130:33631, 127.8.18.129:40911
rpc_server_allow_ephemeral_ports | true | unsafe | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_format | pb | hidden | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-0/data/info.pb | hidden | 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-1/data/info.pb | hidden | 127.8.18.130:33631
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+--------------------+----------------------------------------
builtin_ntp_servers | 127.8.18.148:42749 | 127.8.18.130:33631, 127.8.18.129:40911
time_source | builtin | 127.8.18.130:33631, 127.8.18.129:40911
Version Summary
Version | Servers
-----------------+-----------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.8.18.190:40323, tserver@127.8.18.130:33631, tserver@127.8.18.129:40911
Tablet Summary
Tablet 1f99182d93ed483e9f520bf312a591e3 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
All reported replicas are:
A = d06d8f442c824758ae944ffe71e3cb44
B = 01cff8a283e94658aea5fad9fa0b2db4
C = 747982f4b834445b93f077dd1295b2ff
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | -1 | Yes
C | A B* C | 1 | -1 | Yes
Tablet 459e1d55c65e4857a7ccba6b13316243 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet b29f930721b943039d1df1f03e073a74 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | -1 | Yes
B | A B* C | 2 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:29:52.650406 8266 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:29:52.818498 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 151: this message will repeat every 5th retry.
I20260810 22:29:53.705016 8670 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20260810 22:29:53.705078 8486 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+--------------------+---------
e52d1e743df1434f88d46b1016be2ae7 | 127.8.18.190:40323 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.8.18.148:42749 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+--------------------+-------------+----------+----------------+-----------------
01cff8a283e94658aea5fad9fa0b2db4 | 127.8.18.130:33631 | HEALTHY | <none> | 3 | 0
747982f4b834445b93f077dd1295b2ff | 127.8.18.129:40911 | HEALTHY | <none> | 0 | 0
d06d8f442c824758ae944ffe71e3cb44 | 127.8.18.131:36507 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.8.18.131:36507: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+----------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.8.18.130:33631, 127.8.18.129:40911
ipki_server_key_size | 768 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.129 | experimental | 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.130 | experimental | 127.8.18.130:33631
never_fsync | true | unsafe,advanced | 127.8.18.130:33631, 127.8.18.129:40911
openssl_security_level_override | 0 | unsafe,hidden | 127.8.18.130:33631, 127.8.18.129:40911
rpc_server_allow_ephemeral_ports | true | unsafe | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_format | pb | hidden | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-0/data/info.pb | hidden | 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-1/data/info.pb | hidden | 127.8.18.130:33631
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+--------------------+----------------------------------------
builtin_ntp_servers | 127.8.18.148:42749 | 127.8.18.130:33631, 127.8.18.129:40911
time_source | builtin | 127.8.18.130:33631, 127.8.18.129:40911
Version Summary
Version | Servers
-----------------+-----------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.8.18.190:40323, tserver@127.8.18.129:40911, tserver@127.8.18.130:33631
Tablet Summary
Tablet 1f99182d93ed483e9f520bf312a591e3 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
All reported replicas are:
A = d06d8f442c824758ae944ffe71e3cb44
B = 01cff8a283e94658aea5fad9fa0b2db4
C = 747982f4b834445b93f077dd1295b2ff
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | -1 | Yes
C | A B* C | 1 | -1 | Yes
Tablet 459e1d55c65e4857a7ccba6b13316243 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet b29f930721b943039d1df1f03e073a74 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | -1 | Yes
B | A B* C | 2 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:29:53.724936 8266 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:29:54.297429 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 151: this message will repeat every 5th retry.
I20260810 22:29:54.779089 8670 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20260810 22:29:54.779140 8486 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+--------------------+---------
e52d1e743df1434f88d46b1016be2ae7 | 127.8.18.190:40323 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.8.18.148:42749 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+--------------------+-------------+----------+----------------+-----------------
01cff8a283e94658aea5fad9fa0b2db4 | 127.8.18.130:33631 | HEALTHY | <none> | 3 | 0
747982f4b834445b93f077dd1295b2ff | 127.8.18.129:40911 | HEALTHY | <none> | 0 | 0
d06d8f442c824758ae944ffe71e3cb44 | 127.8.18.131:36507 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.8.18.131:36507: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+----------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.8.18.130:33631, 127.8.18.129:40911
ipki_server_key_size | 768 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.129 | experimental | 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.130 | experimental | 127.8.18.130:33631
never_fsync | true | unsafe,advanced | 127.8.18.130:33631, 127.8.18.129:40911
openssl_security_level_override | 0 | unsafe,hidden | 127.8.18.130:33631, 127.8.18.129:40911
rpc_server_allow_ephemeral_ports | true | unsafe | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_format | pb | hidden | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-0/data/info.pb | hidden | 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-1/data/info.pb | hidden | 127.8.18.130:33631
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+--------------------+----------------------------------------
builtin_ntp_servers | 127.8.18.148:42749 | 127.8.18.130:33631, 127.8.18.129:40911
time_source | builtin | 127.8.18.130:33631, 127.8.18.129:40911
Version Summary
Version | Servers
-----------------+-----------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.8.18.190:40323, tserver@127.8.18.130:33631, tserver@127.8.18.129:40911
Tablet Summary
Tablet 1f99182d93ed483e9f520bf312a591e3 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
All reported replicas are:
A = d06d8f442c824758ae944ffe71e3cb44
B = 01cff8a283e94658aea5fad9fa0b2db4
C = 747982f4b834445b93f077dd1295b2ff
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | -1 | Yes
C | A B* C | 1 | -1 | Yes
Tablet 459e1d55c65e4857a7ccba6b13316243 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet b29f930721b943039d1df1f03e073a74 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | -1 | Yes
B | A B* C | 2 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:29:54.803200 8266 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:29:54.887737 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 151: this message will repeat every 5th retry.
W20260810 22:29:55.192202 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 156: this message will repeat every 5th retry.
W20260810 22:29:55.441484 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 31 similar messages]
I20260810 22:29:55.859717 8670 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20260810 22:29:55.860800 8486 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+--------------------+---------
e52d1e743df1434f88d46b1016be2ae7 | 127.8.18.190:40323 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.8.18.148:42749 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+--------------------+-------------+----------+----------------+-----------------
01cff8a283e94658aea5fad9fa0b2db4 | 127.8.18.130:33631 | HEALTHY | <none> | 3 | 0
747982f4b834445b93f077dd1295b2ff | 127.8.18.129:40911 | HEALTHY | <none> | 0 | 0
d06d8f442c824758ae944ffe71e3cb44 | 127.8.18.131:36507 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.8.18.131:36507: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+----------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.8.18.130:33631, 127.8.18.129:40911
ipki_server_key_size | 768 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.129 | experimental | 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.130 | experimental | 127.8.18.130:33631
never_fsync | true | unsafe,advanced | 127.8.18.130:33631, 127.8.18.129:40911
openssl_security_level_override | 0 | unsafe,hidden | 127.8.18.130:33631, 127.8.18.129:40911
rpc_server_allow_ephemeral_ports | true | unsafe | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_format | pb | hidden | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-0/data/info.pb | hidden | 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-1/data/info.pb | hidden | 127.8.18.130:33631
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+--------------------+----------------------------------------
builtin_ntp_servers | 127.8.18.148:42749 | 127.8.18.130:33631, 127.8.18.129:40911
time_source | builtin | 127.8.18.130:33631, 127.8.18.129:40911
Version Summary
Version | Servers
-----------------+-----------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.8.18.190:40323, tserver@127.8.18.130:33631, tserver@127.8.18.129:40911
Tablet Summary
Tablet 1f99182d93ed483e9f520bf312a591e3 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
All reported replicas are:
A = d06d8f442c824758ae944ffe71e3cb44
B = 01cff8a283e94658aea5fad9fa0b2db4
C = 747982f4b834445b93f077dd1295b2ff
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | -1 | Yes
C | A B* C | 1 | -1 | Yes
Tablet 459e1d55c65e4857a7ccba6b13316243 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet b29f930721b943039d1df1f03e073a74 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | -1 | Yes
B | A B* C | 2 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:29:55.878890 8266 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
I20260810 22:29:56.935326 8670 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20260810 22:29:56.935869 8486 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+--------------------+---------
e52d1e743df1434f88d46b1016be2ae7 | 127.8.18.190:40323 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.8.18.148:42749 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+--------------------+-------------+----------+----------------+-----------------
01cff8a283e94658aea5fad9fa0b2db4 | 127.8.18.130:33631 | HEALTHY | <none> | 3 | 0
747982f4b834445b93f077dd1295b2ff | 127.8.18.129:40911 | HEALTHY | <none> | 0 | 0
d06d8f442c824758ae944ffe71e3cb44 | 127.8.18.131:36507 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.8.18.131:36507: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+----------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.8.18.130:33631, 127.8.18.129:40911
ipki_server_key_size | 768 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.129 | experimental | 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.130 | experimental | 127.8.18.130:33631
never_fsync | true | unsafe,advanced | 127.8.18.130:33631, 127.8.18.129:40911
openssl_security_level_override | 0 | unsafe,hidden | 127.8.18.130:33631, 127.8.18.129:40911
rpc_server_allow_ephemeral_ports | true | unsafe | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_format | pb | hidden | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-0/data/info.pb | hidden | 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-1/data/info.pb | hidden | 127.8.18.130:33631
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+--------------------+----------------------------------------
builtin_ntp_servers | 127.8.18.148:42749 | 127.8.18.130:33631, 127.8.18.129:40911
time_source | builtin | 127.8.18.130:33631, 127.8.18.129:40911
Version Summary
Version | Servers
-----------------+-----------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.8.18.190:40323, tserver@127.8.18.130:33631, tserver@127.8.18.129:40911
Tablet Summary
Tablet 1f99182d93ed483e9f520bf312a591e3 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
All reported replicas are:
A = d06d8f442c824758ae944ffe71e3cb44
B = 01cff8a283e94658aea5fad9fa0b2db4
C = 747982f4b834445b93f077dd1295b2ff
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | -1 | Yes
C | A B* C | 1 | -1 | Yes
Tablet 459e1d55c65e4857a7ccba6b13316243 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet b29f930721b943039d1df1f03e073a74 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | -1 | Yes
B | A B* C | 2 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:29:56.956698 8266 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:29:57.053996 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 156: this message will repeat every 5th retry.
W20260810 22:29:57.450409 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 156: this message will repeat every 5th retry.
W20260810 22:29:57.875932 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 161: this message will repeat every 5th retry.
I20260810 22:29:58.014227 8670 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20260810 22:29:58.014788 8486 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+--------------------+---------
e52d1e743df1434f88d46b1016be2ae7 | 127.8.18.190:40323 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.8.18.148:42749 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+--------------------+-------------+----------+----------------+-----------------
01cff8a283e94658aea5fad9fa0b2db4 | 127.8.18.130:33631 | HEALTHY | <none> | 3 | 0
747982f4b834445b93f077dd1295b2ff | 127.8.18.129:40911 | HEALTHY | <none> | 0 | 0
d06d8f442c824758ae944ffe71e3cb44 | 127.8.18.131:36507 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.8.18.131:36507: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+----------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.8.18.130:33631, 127.8.18.129:40911
ipki_server_key_size | 768 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.129 | experimental | 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.130 | experimental | 127.8.18.130:33631
never_fsync | true | unsafe,advanced | 127.8.18.130:33631, 127.8.18.129:40911
openssl_security_level_override | 0 | unsafe,hidden | 127.8.18.130:33631, 127.8.18.129:40911
rpc_server_allow_ephemeral_ports | true | unsafe | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_format | pb | hidden | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-0/data/info.pb | hidden | 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-1/data/info.pb | hidden | 127.8.18.130:33631
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+--------------------+----------------------------------------
builtin_ntp_servers | 127.8.18.148:42749 | 127.8.18.130:33631, 127.8.18.129:40911
time_source | builtin | 127.8.18.130:33631, 127.8.18.129:40911
Version Summary
Version | Servers
-----------------+-----------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.8.18.190:40323, tserver@127.8.18.130:33631, tserver@127.8.18.129:40911
Tablet Summary
Tablet 1f99182d93ed483e9f520bf312a591e3 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
All reported replicas are:
A = d06d8f442c824758ae944ffe71e3cb44
B = 01cff8a283e94658aea5fad9fa0b2db4
C = 747982f4b834445b93f077dd1295b2ff
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | -1 | Yes
C | A B* C | 1 | -1 | Yes
Tablet 459e1d55c65e4857a7ccba6b13316243 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet b29f930721b943039d1df1f03e073a74 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | -1 | Yes
B | A B* C | 2 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:29:58.036402 8266 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
I20260810 22:29:59.092285 8486 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20260810 22:29:59.094841 8670 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+--------------------+---------
e52d1e743df1434f88d46b1016be2ae7 | 127.8.18.190:40323 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.8.18.148:42749 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+--------------------+-------------+----------+----------------+-----------------
01cff8a283e94658aea5fad9fa0b2db4 | 127.8.18.130:33631 | HEALTHY | <none> | 3 | 0
747982f4b834445b93f077dd1295b2ff | 127.8.18.129:40911 | HEALTHY | <none> | 0 | 0
d06d8f442c824758ae944ffe71e3cb44 | 127.8.18.131:36507 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.8.18.131:36507: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+----------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.8.18.130:33631, 127.8.18.129:40911
ipki_server_key_size | 768 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.129 | experimental | 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.130 | experimental | 127.8.18.130:33631
never_fsync | true | unsafe,advanced | 127.8.18.130:33631, 127.8.18.129:40911
openssl_security_level_override | 0 | unsafe,hidden | 127.8.18.130:33631, 127.8.18.129:40911
rpc_server_allow_ephemeral_ports | true | unsafe | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_format | pb | hidden | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-0/data/info.pb | hidden | 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-1/data/info.pb | hidden | 127.8.18.130:33631
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+--------------------+----------------------------------------
builtin_ntp_servers | 127.8.18.148:42749 | 127.8.18.130:33631, 127.8.18.129:40911
time_source | builtin | 127.8.18.130:33631, 127.8.18.129:40911
Version Summary
Version | Servers
-----------------+-----------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.8.18.190:40323, tserver@127.8.18.129:40911, tserver@127.8.18.130:33631
Tablet Summary
Tablet 1f99182d93ed483e9f520bf312a591e3 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
All reported replicas are:
A = d06d8f442c824758ae944ffe71e3cb44
B = 01cff8a283e94658aea5fad9fa0b2db4
C = 747982f4b834445b93f077dd1295b2ff
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | -1 | Yes
C | A B* C | 1 | -1 | Yes
Tablet 459e1d55c65e4857a7ccba6b13316243 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet b29f930721b943039d1df1f03e073a74 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | -1 | Yes
B | A B* C | 2 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:29:59.114666 8266 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:29:59.352203 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 161: this message will repeat every 5th retry.
W20260810 22:30:00.119670 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 161: this message will repeat every 5th retry.
I20260810 22:30:00.168680 8670 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20260810 22:30:00.171172 8486 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+--------------------+---------
e52d1e743df1434f88d46b1016be2ae7 | 127.8.18.190:40323 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.8.18.148:42749 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+--------------------+-------------+----------+----------------+-----------------
01cff8a283e94658aea5fad9fa0b2db4 | 127.8.18.130:33631 | HEALTHY | <none> | 3 | 0
747982f4b834445b93f077dd1295b2ff | 127.8.18.129:40911 | HEALTHY | <none> | 0 | 0
d06d8f442c824758ae944ffe71e3cb44 | 127.8.18.131:36507 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.8.18.131:36507: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+----------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.8.18.130:33631, 127.8.18.129:40911
ipki_server_key_size | 768 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.129 | experimental | 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.130 | experimental | 127.8.18.130:33631
never_fsync | true | unsafe,advanced | 127.8.18.130:33631, 127.8.18.129:40911
openssl_security_level_override | 0 | unsafe,hidden | 127.8.18.130:33631, 127.8.18.129:40911
rpc_server_allow_ephemeral_ports | true | unsafe | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_format | pb | hidden | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-0/data/info.pb | hidden | 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-1/data/info.pb | hidden | 127.8.18.130:33631
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+--------------------+----------------------------------------
builtin_ntp_servers | 127.8.18.148:42749 | 127.8.18.130:33631, 127.8.18.129:40911
time_source | builtin | 127.8.18.130:33631, 127.8.18.129:40911
Version Summary
Version | Servers
-----------------+-----------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.8.18.190:40323, tserver@127.8.18.130:33631, tserver@127.8.18.129:40911
Tablet Summary
Tablet 1f99182d93ed483e9f520bf312a591e3 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
All reported replicas are:
A = d06d8f442c824758ae944ffe71e3cb44
B = 01cff8a283e94658aea5fad9fa0b2db4
C = 747982f4b834445b93f077dd1295b2ff
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | -1 | Yes
C | A B* C | 1 | -1 | Yes
Tablet 459e1d55c65e4857a7ccba6b13316243 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet b29f930721b943039d1df1f03e073a74 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | -1 | Yes
B | A B* C | 2 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:30:00.190277 8266 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:30:00.214426 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 166: this message will repeat every 5th retry.
W20260810 22:30:00.443328 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 28 similar messages]
I20260810 22:30:01.245064 8670 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20260810 22:30:01.245486 8486 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+--------------------+---------
e52d1e743df1434f88d46b1016be2ae7 | 127.8.18.190:40323 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.8.18.148:42749 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+--------------------+-------------+----------+----------------+-----------------
01cff8a283e94658aea5fad9fa0b2db4 | 127.8.18.130:33631 | HEALTHY | <none> | 3 | 0
747982f4b834445b93f077dd1295b2ff | 127.8.18.129:40911 | HEALTHY | <none> | 0 | 0
d06d8f442c824758ae944ffe71e3cb44 | 127.8.18.131:36507 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.8.18.131:36507: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+----------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.8.18.130:33631, 127.8.18.129:40911
ipki_server_key_size | 768 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.129 | experimental | 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.130 | experimental | 127.8.18.130:33631
never_fsync | true | unsafe,advanced | 127.8.18.130:33631, 127.8.18.129:40911
openssl_security_level_override | 0 | unsafe,hidden | 127.8.18.130:33631, 127.8.18.129:40911
rpc_server_allow_ephemeral_ports | true | unsafe | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_format | pb | hidden | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-0/data/info.pb | hidden | 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-1/data/info.pb | hidden | 127.8.18.130:33631
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+--------------------+----------------------------------------
builtin_ntp_servers | 127.8.18.148:42749 | 127.8.18.130:33631, 127.8.18.129:40911
time_source | builtin | 127.8.18.130:33631, 127.8.18.129:40911
Version Summary
Version | Servers
-----------------+-----------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.8.18.190:40323, tserver@127.8.18.130:33631, tserver@127.8.18.129:40911
Tablet Summary
Tablet 1f99182d93ed483e9f520bf312a591e3 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
All reported replicas are:
A = d06d8f442c824758ae944ffe71e3cb44
B = 01cff8a283e94658aea5fad9fa0b2db4
C = 747982f4b834445b93f077dd1295b2ff
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | -1 | Yes
C | A B* C | 1 | -1 | Yes
Tablet 459e1d55c65e4857a7ccba6b13316243 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet b29f930721b943039d1df1f03e073a74 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | -1 | Yes
B | A B* C | 2 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:30:01.266086 8266 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:30:01.998438 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 166: this message will repeat every 5th retry.
I20260810 22:30:02.319715 8670 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20260810 22:30:02.320259 8486 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+--------------------+---------
e52d1e743df1434f88d46b1016be2ae7 | 127.8.18.190:40323 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.8.18.148:42749 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+--------------------+-------------+----------+----------------+-----------------
01cff8a283e94658aea5fad9fa0b2db4 | 127.8.18.130:33631 | HEALTHY | <none> | 3 | 0
747982f4b834445b93f077dd1295b2ff | 127.8.18.129:40911 | HEALTHY | <none> | 0 | 0
d06d8f442c824758ae944ffe71e3cb44 | 127.8.18.131:36507 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.8.18.131:36507: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+----------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.8.18.130:33631, 127.8.18.129:40911
ipki_server_key_size | 768 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.129 | experimental | 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.130 | experimental | 127.8.18.130:33631
never_fsync | true | unsafe,advanced | 127.8.18.130:33631, 127.8.18.129:40911
openssl_security_level_override | 0 | unsafe,hidden | 127.8.18.130:33631, 127.8.18.129:40911
rpc_server_allow_ephemeral_ports | true | unsafe | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_format | pb | hidden | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-0/data/info.pb | hidden | 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-1/data/info.pb | hidden | 127.8.18.130:33631
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+--------------------+----------------------------------------
builtin_ntp_servers | 127.8.18.148:42749 | 127.8.18.130:33631, 127.8.18.129:40911
time_source | builtin | 127.8.18.130:33631, 127.8.18.129:40911
Version Summary
Version | Servers
-----------------+-----------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.8.18.190:40323, tserver@127.8.18.130:33631, tserver@127.8.18.129:40911
Tablet Summary
Tablet 1f99182d93ed483e9f520bf312a591e3 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
All reported replicas are:
A = d06d8f442c824758ae944ffe71e3cb44
B = 01cff8a283e94658aea5fad9fa0b2db4
C = 747982f4b834445b93f077dd1295b2ff
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | -1 | Yes
C | A B* C | 1 | -1 | Yes
Tablet 459e1d55c65e4857a7ccba6b13316243 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet b29f930721b943039d1df1f03e073a74 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | -1 | Yes
B | A B* C | 2 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:30:02.342795 8266 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:30:02.745124 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 171: this message will repeat every 5th retry.
W20260810 22:30:02.854465 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 166: this message will repeat every 5th retry.
I20260810 22:30:03.398253 8486 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20260810 22:30:03.398253 8670 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+--------------------+---------
e52d1e743df1434f88d46b1016be2ae7 | 127.8.18.190:40323 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.8.18.148:42749 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+--------------------+-------------+----------+----------------+-----------------
01cff8a283e94658aea5fad9fa0b2db4 | 127.8.18.130:33631 | HEALTHY | <none> | 3 | 0
747982f4b834445b93f077dd1295b2ff | 127.8.18.129:40911 | HEALTHY | <none> | 0 | 0
d06d8f442c824758ae944ffe71e3cb44 | 127.8.18.131:36507 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.8.18.131:36507: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+----------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.8.18.130:33631, 127.8.18.129:40911
ipki_server_key_size | 768 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.129 | experimental | 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.130 | experimental | 127.8.18.130:33631
never_fsync | true | unsafe,advanced | 127.8.18.130:33631, 127.8.18.129:40911
openssl_security_level_override | 0 | unsafe,hidden | 127.8.18.130:33631, 127.8.18.129:40911
rpc_server_allow_ephemeral_ports | true | unsafe | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_format | pb | hidden | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-0/data/info.pb | hidden | 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-1/data/info.pb | hidden | 127.8.18.130:33631
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+--------------------+----------------------------------------
builtin_ntp_servers | 127.8.18.148:42749 | 127.8.18.130:33631, 127.8.18.129:40911
time_source | builtin | 127.8.18.130:33631, 127.8.18.129:40911
Version Summary
Version | Servers
-----------------+-----------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.8.18.190:40323, tserver@127.8.18.129:40911, tserver@127.8.18.130:33631
Tablet Summary
Tablet 1f99182d93ed483e9f520bf312a591e3 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
All reported replicas are:
A = d06d8f442c824758ae944ffe71e3cb44
B = 01cff8a283e94658aea5fad9fa0b2db4
C = 747982f4b834445b93f077dd1295b2ff
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | -1 | Yes
C | A B* C | 1 | -1 | Yes
Tablet 459e1d55c65e4857a7ccba6b13316243 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet b29f930721b943039d1df1f03e073a74 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | -1 | Yes
B | A B* C | 2 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:30:03.418272 8266 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
I20260810 22:30:04.472792 8670 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20260810 22:30:04.473789 8486 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+--------------------+---------
e52d1e743df1434f88d46b1016be2ae7 | 127.8.18.190:40323 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.8.18.148:42749 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+--------------------+-------------+----------+----------------+-----------------
01cff8a283e94658aea5fad9fa0b2db4 | 127.8.18.130:33631 | HEALTHY | <none> | 3 | 0
747982f4b834445b93f077dd1295b2ff | 127.8.18.129:40911 | HEALTHY | <none> | 0 | 0
d06d8f442c824758ae944ffe71e3cb44 | 127.8.18.131:36507 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.8.18.131:36507: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+----------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.8.18.130:33631, 127.8.18.129:40911
ipki_server_key_size | 768 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.129 | experimental | 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.130 | experimental | 127.8.18.130:33631
never_fsync | true | unsafe,advanced | 127.8.18.130:33631, 127.8.18.129:40911
openssl_security_level_override | 0 | unsafe,hidden | 127.8.18.130:33631, 127.8.18.129:40911
rpc_server_allow_ephemeral_ports | true | unsafe | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_format | pb | hidden | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-0/data/info.pb | hidden | 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-1/data/info.pb | hidden | 127.8.18.130:33631
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+--------------------+----------------------------------------
builtin_ntp_servers | 127.8.18.148:42749 | 127.8.18.130:33631, 127.8.18.129:40911
time_source | builtin | 127.8.18.130:33631, 127.8.18.129:40911
Version Summary
Version | Servers
-----------------+-----------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.8.18.190:40323, tserver@127.8.18.130:33631, tserver@127.8.18.129:40911
Tablet Summary
Tablet 1f99182d93ed483e9f520bf312a591e3 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
All reported replicas are:
A = d06d8f442c824758ae944ffe71e3cb44
B = 01cff8a283e94658aea5fad9fa0b2db4
C = 747982f4b834445b93f077dd1295b2ff
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | -1 | Yes
C | A B* C | 1 | -1 | Yes
Tablet 459e1d55c65e4857a7ccba6b13316243 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet b29f930721b943039d1df1f03e073a74 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | -1 | Yes
B | A B* C | 2 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:30:04.497143 8266 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:30:04.612767 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 171: this message will repeat every 5th retry.
W20260810 22:30:05.111516 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 171: this message will repeat every 5th retry.
W20260810 22:30:05.365041 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 176: this message will repeat every 5th retry.
W20260810 22:30:05.501905 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 29 similar messages]
I20260810 22:30:05.551226 8670 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20260810 22:30:05.551273 8486 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+--------------------+---------
e52d1e743df1434f88d46b1016be2ae7 | 127.8.18.190:40323 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.8.18.148:42749 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+--------------------+-------------+----------+----------------+-----------------
01cff8a283e94658aea5fad9fa0b2db4 | 127.8.18.130:33631 | HEALTHY | <none> | 3 | 0
747982f4b834445b93f077dd1295b2ff | 127.8.18.129:40911 | HEALTHY | <none> | 0 | 0
d06d8f442c824758ae944ffe71e3cb44 | 127.8.18.131:36507 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.8.18.131:36507: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+----------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.8.18.130:33631, 127.8.18.129:40911
ipki_server_key_size | 768 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.129 | experimental | 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.130 | experimental | 127.8.18.130:33631
never_fsync | true | unsafe,advanced | 127.8.18.130:33631, 127.8.18.129:40911
openssl_security_level_override | 0 | unsafe,hidden | 127.8.18.130:33631, 127.8.18.129:40911
rpc_server_allow_ephemeral_ports | true | unsafe | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_format | pb | hidden | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-0/data/info.pb | hidden | 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-1/data/info.pb | hidden | 127.8.18.130:33631
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+--------------------+----------------------------------------
builtin_ntp_servers | 127.8.18.148:42749 | 127.8.18.130:33631, 127.8.18.129:40911
time_source | builtin | 127.8.18.130:33631, 127.8.18.129:40911
Version Summary
Version | Servers
-----------------+-----------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.8.18.190:40323, tserver@127.8.18.130:33631, tserver@127.8.18.129:40911
Tablet Summary
Tablet 1f99182d93ed483e9f520bf312a591e3 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
All reported replicas are:
A = d06d8f442c824758ae944ffe71e3cb44
B = 01cff8a283e94658aea5fad9fa0b2db4
C = 747982f4b834445b93f077dd1295b2ff
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | -1 | Yes
C | A B* C | 1 | -1 | Yes
Tablet 459e1d55c65e4857a7ccba6b13316243 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet b29f930721b943039d1df1f03e073a74 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | -1 | Yes
B | A B* C | 2 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:30:05.570029 8266 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
I20260810 22:30:06.628611 8486 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20260810 22:30:06.628623 8670 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+--------------------+---------
e52d1e743df1434f88d46b1016be2ae7 | 127.8.18.190:40323 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.8.18.148:42749 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+--------------------+-------------+----------+----------------+-----------------
01cff8a283e94658aea5fad9fa0b2db4 | 127.8.18.130:33631 | HEALTHY | <none> | 3 | 0
747982f4b834445b93f077dd1295b2ff | 127.8.18.129:40911 | HEALTHY | <none> | 0 | 0
d06d8f442c824758ae944ffe71e3cb44 | 127.8.18.131:36507 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.8.18.131:36507: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+----------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.8.18.130:33631, 127.8.18.129:40911
ipki_server_key_size | 768 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.129 | experimental | 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.130 | experimental | 127.8.18.130:33631
never_fsync | true | unsafe,advanced | 127.8.18.130:33631, 127.8.18.129:40911
openssl_security_level_override | 0 | unsafe,hidden | 127.8.18.130:33631, 127.8.18.129:40911
rpc_server_allow_ephemeral_ports | true | unsafe | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_format | pb | hidden | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-0/data/info.pb | hidden | 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-1/data/info.pb | hidden | 127.8.18.130:33631
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+--------------------+----------------------------------------
builtin_ntp_servers | 127.8.18.148:42749 | 127.8.18.130:33631, 127.8.18.129:40911
time_source | builtin | 127.8.18.130:33631, 127.8.18.129:40911
Version Summary
Version | Servers
-----------------+-----------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.8.18.190:40323, tserver@127.8.18.129:40911, tserver@127.8.18.130:33631
Tablet Summary
Tablet 1f99182d93ed483e9f520bf312a591e3 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
All reported replicas are:
A = d06d8f442c824758ae944ffe71e3cb44
B = 01cff8a283e94658aea5fad9fa0b2db4
C = 747982f4b834445b93f077dd1295b2ff
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | -1 | Yes
C | A B* C | 1 | -1 | Yes
Tablet 459e1d55c65e4857a7ccba6b13316243 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet b29f930721b943039d1df1f03e073a74 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | -1 | Yes
B | A B* C | 2 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:30:06.650312 8266 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:30:06.822028 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 176: this message will repeat every 5th retry.
W20260810 22:30:07.578215 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 176: this message will repeat every 5th retry.
I20260810 22:30:07.708045 8670 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20260810 22:30:07.708376 8486 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+--------------------+---------
e52d1e743df1434f88d46b1016be2ae7 | 127.8.18.190:40323 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.8.18.148:42749 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+--------------------+-------------+----------+----------------+-----------------
01cff8a283e94658aea5fad9fa0b2db4 | 127.8.18.130:33631 | HEALTHY | <none> | 3 | 0
747982f4b834445b93f077dd1295b2ff | 127.8.18.129:40911 | HEALTHY | <none> | 0 | 0
d06d8f442c824758ae944ffe71e3cb44 | 127.8.18.131:36507 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.8.18.131:36507: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+----------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.8.18.130:33631, 127.8.18.129:40911
ipki_server_key_size | 768 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.129 | experimental | 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.130 | experimental | 127.8.18.130:33631
never_fsync | true | unsafe,advanced | 127.8.18.130:33631, 127.8.18.129:40911
openssl_security_level_override | 0 | unsafe,hidden | 127.8.18.130:33631, 127.8.18.129:40911
rpc_server_allow_ephemeral_ports | true | unsafe | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_format | pb | hidden | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-0/data/info.pb | hidden | 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-1/data/info.pb | hidden | 127.8.18.130:33631
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+--------------------+----------------------------------------
builtin_ntp_servers | 127.8.18.148:42749 | 127.8.18.130:33631, 127.8.18.129:40911
time_source | builtin | 127.8.18.130:33631, 127.8.18.129:40911
Version Summary
Version | Servers
-----------------+-----------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.8.18.190:40323, tserver@127.8.18.130:33631, tserver@127.8.18.129:40911
Tablet Summary
Tablet 1f99182d93ed483e9f520bf312a591e3 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
All reported replicas are:
A = d06d8f442c824758ae944ffe71e3cb44
B = 01cff8a283e94658aea5fad9fa0b2db4
C = 747982f4b834445b93f077dd1295b2ff
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | -1 | Yes
C | A B* C | 1 | -1 | Yes
Tablet 459e1d55c65e4857a7ccba6b13316243 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet b29f930721b943039d1df1f03e073a74 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | -1 | Yes
B | A B* C | 2 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:30:07.733048 8266 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:30:07.750942 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 181: this message will repeat every 5th retry.
I20260810 22:30:08.786545 8670 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20260810 22:30:08.786878 8486 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+--------------------+---------
e52d1e743df1434f88d46b1016be2ae7 | 127.8.18.190:40323 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.8.18.148:42749 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+--------------------+-------------+----------+----------------+-----------------
01cff8a283e94658aea5fad9fa0b2db4 | 127.8.18.130:33631 | HEALTHY | <none> | 3 | 0
747982f4b834445b93f077dd1295b2ff | 127.8.18.129:40911 | HEALTHY | <none> | 0 | 0
d06d8f442c824758ae944ffe71e3cb44 | 127.8.18.131:36507 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.8.18.131:36507: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+----------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.8.18.130:33631, 127.8.18.129:40911
ipki_server_key_size | 768 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.129 | experimental | 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.130 | experimental | 127.8.18.130:33631
never_fsync | true | unsafe,advanced | 127.8.18.130:33631, 127.8.18.129:40911
openssl_security_level_override | 0 | unsafe,hidden | 127.8.18.130:33631, 127.8.18.129:40911
rpc_server_allow_ephemeral_ports | true | unsafe | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_format | pb | hidden | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-0/data/info.pb | hidden | 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-1/data/info.pb | hidden | 127.8.18.130:33631
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+--------------------+----------------------------------------
builtin_ntp_servers | 127.8.18.148:42749 | 127.8.18.130:33631, 127.8.18.129:40911
time_source | builtin | 127.8.18.130:33631, 127.8.18.129:40911
Version Summary
Version | Servers
-----------------+-----------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.8.18.190:40323, tserver@127.8.18.130:33631, tserver@127.8.18.129:40911
Tablet Summary
Tablet 1f99182d93ed483e9f520bf312a591e3 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
All reported replicas are:
A = d06d8f442c824758ae944ffe71e3cb44
B = 01cff8a283e94658aea5fad9fa0b2db4
C = 747982f4b834445b93f077dd1295b2ff
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | -1 | Yes
C | A B* C | 1 | -1 | Yes
Tablet 459e1d55c65e4857a7ccba6b13316243 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet b29f930721b943039d1df1f03e073a74 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | -1 | Yes
B | A B* C | 2 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:30:08.811695 8266 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:30:09.510106 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 181: this message will repeat every 5th retry.
I20260810 22:30:09.863240 8670 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20260810 22:30:09.863807 8486 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+--------------------+---------
e52d1e743df1434f88d46b1016be2ae7 | 127.8.18.190:40323 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.8.18.148:42749 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+--------------------+-------------+----------+----------------+-----------------
01cff8a283e94658aea5fad9fa0b2db4 | 127.8.18.130:33631 | HEALTHY | <none> | 3 | 0
747982f4b834445b93f077dd1295b2ff | 127.8.18.129:40911 | HEALTHY | <none> | 0 | 0
d06d8f442c824758ae944ffe71e3cb44 | 127.8.18.131:36507 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.8.18.131:36507: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+----------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.8.18.130:33631, 127.8.18.129:40911
ipki_server_key_size | 768 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.129 | experimental | 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.130 | experimental | 127.8.18.130:33631
never_fsync | true | unsafe,advanced | 127.8.18.130:33631, 127.8.18.129:40911
openssl_security_level_override | 0 | unsafe,hidden | 127.8.18.130:33631, 127.8.18.129:40911
rpc_server_allow_ephemeral_ports | true | unsafe | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_format | pb | hidden | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-0/data/info.pb | hidden | 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-1/data/info.pb | hidden | 127.8.18.130:33631
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+--------------------+----------------------------------------
builtin_ntp_servers | 127.8.18.148:42749 | 127.8.18.130:33631, 127.8.18.129:40911
time_source | builtin | 127.8.18.130:33631, 127.8.18.129:40911
Version Summary
Version | Servers
-----------------+-----------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.8.18.190:40323, tserver@127.8.18.130:33631, tserver@127.8.18.129:40911
Tablet Summary
Tablet 1f99182d93ed483e9f520bf312a591e3 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
All reported replicas are:
A = d06d8f442c824758ae944ffe71e3cb44
B = 01cff8a283e94658aea5fad9fa0b2db4
C = 747982f4b834445b93f077dd1295b2ff
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | -1 | Yes
C | A B* C | 1 | -1 | Yes
Tablet 459e1d55c65e4857a7ccba6b13316243 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet b29f930721b943039d1df1f03e073a74 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | -1 | Yes
B | A B* C | 2 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:30:09.886029 8266 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:30:09.954862 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 181: this message will repeat every 5th retry.
W20260810 22:30:10.368384 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 186: this message will repeat every 5th retry.
W20260810 22:30:10.861671 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 31 similar messages]
I20260810 22:30:10.939872 8486 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20260810 22:30:10.939863 8670 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+--------------------+---------
e52d1e743df1434f88d46b1016be2ae7 | 127.8.18.190:40323 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.8.18.148:42749 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+--------------------+-------------+----------+----------------+-----------------
01cff8a283e94658aea5fad9fa0b2db4 | 127.8.18.130:33631 | HEALTHY | <none> | 3 | 0
747982f4b834445b93f077dd1295b2ff | 127.8.18.129:40911 | HEALTHY | <none> | 0 | 0
d06d8f442c824758ae944ffe71e3cb44 | 127.8.18.131:36507 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.8.18.131:36507: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+----------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.8.18.130:33631, 127.8.18.129:40911
ipki_server_key_size | 768 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.129 | experimental | 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.130 | experimental | 127.8.18.130:33631
never_fsync | true | unsafe,advanced | 127.8.18.130:33631, 127.8.18.129:40911
openssl_security_level_override | 0 | unsafe,hidden | 127.8.18.130:33631, 127.8.18.129:40911
rpc_server_allow_ephemeral_ports | true | unsafe | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_format | pb | hidden | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-0/data/info.pb | hidden | 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-1/data/info.pb | hidden | 127.8.18.130:33631
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+--------------------+----------------------------------------
builtin_ntp_servers | 127.8.18.148:42749 | 127.8.18.130:33631, 127.8.18.129:40911
time_source | builtin | 127.8.18.130:33631, 127.8.18.129:40911
Version Summary
Version | Servers
-----------------+-----------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.8.18.190:40323, tserver@127.8.18.130:33631, tserver@127.8.18.129:40911
Tablet Summary
Tablet 1f99182d93ed483e9f520bf312a591e3 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
All reported replicas are:
A = d06d8f442c824758ae944ffe71e3cb44
B = 01cff8a283e94658aea5fad9fa0b2db4
C = 747982f4b834445b93f077dd1295b2ff
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | -1 | Yes
C | A B* C | 1 | -1 | Yes
Tablet 459e1d55c65e4857a7ccba6b13316243 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet b29f930721b943039d1df1f03e073a74 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | -1 | Yes
B | A B* C | 2 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:30:10.961504 8266 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:30:11.844965 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 186: this message will repeat every 5th retry.
I20260810 22:30:12.018157 8670 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20260810 22:30:12.018208 8486 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+--------------------+---------
e52d1e743df1434f88d46b1016be2ae7 | 127.8.18.190:40323 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.8.18.148:42749 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+--------------------+-------------+----------+----------------+-----------------
01cff8a283e94658aea5fad9fa0b2db4 | 127.8.18.130:33631 | HEALTHY | <none> | 3 | 0
747982f4b834445b93f077dd1295b2ff | 127.8.18.129:40911 | HEALTHY | <none> | 0 | 0
d06d8f442c824758ae944ffe71e3cb44 | 127.8.18.131:36507 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.8.18.131:36507: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+----------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.8.18.130:33631, 127.8.18.129:40911
ipki_server_key_size | 768 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.129 | experimental | 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.130 | experimental | 127.8.18.130:33631
never_fsync | true | unsafe,advanced | 127.8.18.130:33631, 127.8.18.129:40911
openssl_security_level_override | 0 | unsafe,hidden | 127.8.18.130:33631, 127.8.18.129:40911
rpc_server_allow_ephemeral_ports | true | unsafe | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_format | pb | hidden | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-0/data/info.pb | hidden | 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-1/data/info.pb | hidden | 127.8.18.130:33631
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+--------------------+----------------------------------------
builtin_ntp_servers | 127.8.18.148:42749 | 127.8.18.130:33631, 127.8.18.129:40911
time_source | builtin | 127.8.18.130:33631, 127.8.18.129:40911
Version Summary
Version | Servers
-----------------+-----------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.8.18.190:40323, tserver@127.8.18.130:33631, tserver@127.8.18.129:40911
Tablet Summary
Tablet 1f99182d93ed483e9f520bf312a591e3 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
All reported replicas are:
A = d06d8f442c824758ae944ffe71e3cb44
B = 01cff8a283e94658aea5fad9fa0b2db4
C = 747982f4b834445b93f077dd1295b2ff
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | -1 | Yes
C | A B* C | 1 | -1 | Yes
Tablet 459e1d55c65e4857a7ccba6b13316243 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet b29f930721b943039d1df1f03e073a74 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | -1 | Yes
B | A B* C | 2 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:30:12.036937 8266 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:30:12.363653 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 186: this message will repeat every 5th retry.
W20260810 22:30:12.842927 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 191: this message will repeat every 5th retry.
I20260810 22:30:13.091235 8670 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20260810 22:30:13.092993 8486 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+--------------------+---------
e52d1e743df1434f88d46b1016be2ae7 | 127.8.18.190:40323 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.8.18.148:42749 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+--------------------+-------------+----------+----------------+-----------------
01cff8a283e94658aea5fad9fa0b2db4 | 127.8.18.130:33631 | HEALTHY | <none> | 3 | 0
747982f4b834445b93f077dd1295b2ff | 127.8.18.129:40911 | HEALTHY | <none> | 0 | 0
d06d8f442c824758ae944ffe71e3cb44 | 127.8.18.131:36507 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.8.18.131:36507: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+----------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.8.18.130:33631, 127.8.18.129:40911
ipki_server_key_size | 768 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.129 | experimental | 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.130 | experimental | 127.8.18.130:33631
never_fsync | true | unsafe,advanced | 127.8.18.130:33631, 127.8.18.129:40911
openssl_security_level_override | 0 | unsafe,hidden | 127.8.18.130:33631, 127.8.18.129:40911
rpc_server_allow_ephemeral_ports | true | unsafe | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_format | pb | hidden | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-0/data/info.pb | hidden | 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-1/data/info.pb | hidden | 127.8.18.130:33631
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+--------------------+----------------------------------------
builtin_ntp_servers | 127.8.18.148:42749 | 127.8.18.130:33631, 127.8.18.129:40911
time_source | builtin | 127.8.18.130:33631, 127.8.18.129:40911
Version Summary
Version | Servers
-----------------+-----------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.8.18.190:40323, tserver@127.8.18.130:33631, tserver@127.8.18.129:40911
Tablet Summary
Tablet 1f99182d93ed483e9f520bf312a591e3 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
All reported replicas are:
A = d06d8f442c824758ae944ffe71e3cb44
B = 01cff8a283e94658aea5fad9fa0b2db4
C = 747982f4b834445b93f077dd1295b2ff
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | -1 | Yes
C | A B* C | 1 | -1 | Yes
Tablet 459e1d55c65e4857a7ccba6b13316243 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet b29f930721b943039d1df1f03e073a74 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | -1 | Yes
B | A B* C | 2 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:30:13.118687 8266 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
I20260810 22:30:14.171926 8670 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20260810 22:30:14.172220 8486 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+--------------------+---------
e52d1e743df1434f88d46b1016be2ae7 | 127.8.18.190:40323 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.8.18.148:42749 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+--------------------+-------------+----------+----------------+-----------------
01cff8a283e94658aea5fad9fa0b2db4 | 127.8.18.130:33631 | HEALTHY | <none> | 3 | 0
747982f4b834445b93f077dd1295b2ff | 127.8.18.129:40911 | HEALTHY | <none> | 0 | 0
d06d8f442c824758ae944ffe71e3cb44 | 127.8.18.131:36507 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.8.18.131:36507: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+----------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.8.18.130:33631, 127.8.18.129:40911
ipki_server_key_size | 768 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.129 | experimental | 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.130 | experimental | 127.8.18.130:33631
never_fsync | true | unsafe,advanced | 127.8.18.130:33631, 127.8.18.129:40911
openssl_security_level_override | 0 | unsafe,hidden | 127.8.18.130:33631, 127.8.18.129:40911
rpc_server_allow_ephemeral_ports | true | unsafe | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_format | pb | hidden | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-0/data/info.pb | hidden | 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-1/data/info.pb | hidden | 127.8.18.130:33631
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+--------------------+----------------------------------------
builtin_ntp_servers | 127.8.18.148:42749 | 127.8.18.130:33631, 127.8.18.129:40911
time_source | builtin | 127.8.18.130:33631, 127.8.18.129:40911
Version Summary
Version | Servers
-----------------+-----------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.8.18.190:40323, tserver@127.8.18.130:33631, tserver@127.8.18.129:40911
Tablet Summary
Tablet 1f99182d93ed483e9f520bf312a591e3 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
All reported replicas are:
A = d06d8f442c824758ae944ffe71e3cb44
B = 01cff8a283e94658aea5fad9fa0b2db4
C = 747982f4b834445b93f077dd1295b2ff
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | -1 | Yes
C | A B* C | 1 | -1 | Yes
Tablet 459e1d55c65e4857a7ccba6b13316243 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet b29f930721b943039d1df1f03e073a74 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | -1 | Yes
B | A B* C | 2 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:30:14.190814 8266 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:30:14.370950 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 191: this message will repeat every 5th retry.
W20260810 22:30:14.729068 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 191: this message will repeat every 5th retry.
I20260810 22:30:15.244237 8670 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20260810 22:30:15.244526 8486 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+--------------------+---------
e52d1e743df1434f88d46b1016be2ae7 | 127.8.18.190:40323 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.8.18.148:42749 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+--------------------+-------------+----------+----------------+-----------------
01cff8a283e94658aea5fad9fa0b2db4 | 127.8.18.130:33631 | HEALTHY | <none> | 3 | 0
747982f4b834445b93f077dd1295b2ff | 127.8.18.129:40911 | HEALTHY | <none> | 0 | 0
d06d8f442c824758ae944ffe71e3cb44 | 127.8.18.131:36507 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.8.18.131:36507: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+----------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.8.18.130:33631, 127.8.18.129:40911
ipki_server_key_size | 768 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.129 | experimental | 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.130 | experimental | 127.8.18.130:33631
never_fsync | true | unsafe,advanced | 127.8.18.130:33631, 127.8.18.129:40911
openssl_security_level_override | 0 | unsafe,hidden | 127.8.18.130:33631, 127.8.18.129:40911
rpc_server_allow_ephemeral_ports | true | unsafe | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_format | pb | hidden | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-0/data/info.pb | hidden | 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-1/data/info.pb | hidden | 127.8.18.130:33631
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+--------------------+----------------------------------------
builtin_ntp_servers | 127.8.18.148:42749 | 127.8.18.130:33631, 127.8.18.129:40911
time_source | builtin | 127.8.18.130:33631, 127.8.18.129:40911
Version Summary
Version | Servers
-----------------+-----------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.8.18.190:40323, tserver@127.8.18.130:33631, tserver@127.8.18.129:40911
Tablet Summary
Tablet 1f99182d93ed483e9f520bf312a591e3 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
All reported replicas are:
A = d06d8f442c824758ae944ffe71e3cb44
B = 01cff8a283e94658aea5fad9fa0b2db4
C = 747982f4b834445b93f077dd1295b2ff
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | -1 | Yes
C | A B* C | 1 | -1 | Yes
Tablet 459e1d55c65e4857a7ccba6b13316243 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet b29f930721b943039d1df1f03e073a74 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | -1 | Yes
B | A B* C | 2 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:30:15.264566 8266 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:30:15.301599 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 196: this message will repeat every 5th retry.
W20260810 22:30:16.056665 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 31 similar messages]
I20260810 22:30:16.317875 8670 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20260810 22:30:16.319571 8486 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+--------------------+---------
e52d1e743df1434f88d46b1016be2ae7 | 127.8.18.190:40323 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.8.18.148:42749 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+--------------------+-------------+----------+----------------+-----------------
01cff8a283e94658aea5fad9fa0b2db4 | 127.8.18.130:33631 | HEALTHY | <none> | 3 | 0
747982f4b834445b93f077dd1295b2ff | 127.8.18.129:40911 | HEALTHY | <none> | 0 | 0
d06d8f442c824758ae944ffe71e3cb44 | 127.8.18.131:36507 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.8.18.131:36507: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+----------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.8.18.130:33631, 127.8.18.129:40911
ipki_server_key_size | 768 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.129 | experimental | 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.130 | experimental | 127.8.18.130:33631
never_fsync | true | unsafe,advanced | 127.8.18.130:33631, 127.8.18.129:40911
openssl_security_level_override | 0 | unsafe,hidden | 127.8.18.130:33631, 127.8.18.129:40911
rpc_server_allow_ephemeral_ports | true | unsafe | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_format | pb | hidden | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-0/data/info.pb | hidden | 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-1/data/info.pb | hidden | 127.8.18.130:33631
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+--------------------+----------------------------------------
builtin_ntp_servers | 127.8.18.148:42749 | 127.8.18.130:33631, 127.8.18.129:40911
time_source | builtin | 127.8.18.130:33631, 127.8.18.129:40911
Version Summary
Version | Servers
-----------------+-----------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.8.18.190:40323, tserver@127.8.18.130:33631, tserver@127.8.18.129:40911
Tablet Summary
Tablet 1f99182d93ed483e9f520bf312a591e3 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
All reported replicas are:
A = d06d8f442c824758ae944ffe71e3cb44
B = 01cff8a283e94658aea5fad9fa0b2db4
C = 747982f4b834445b93f077dd1295b2ff
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | -1 | Yes
C | A B* C | 1 | -1 | Yes
Tablet 459e1d55c65e4857a7ccba6b13316243 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet b29f930721b943039d1df1f03e073a74 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | -1 | Yes
B | A B* C | 2 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:30:16.342186 8266 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:30:16.985788 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 196: this message will repeat every 5th retry.
W20260810 22:30:17.313253 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 196: this message will repeat every 5th retry.
I20260810 22:30:17.401240 8486 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20260810 22:30:17.401579 8670 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+--------------------+---------
e52d1e743df1434f88d46b1016be2ae7 | 127.8.18.190:40323 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.8.18.148:42749 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+--------------------+-------------+----------+----------------+-----------------
01cff8a283e94658aea5fad9fa0b2db4 | 127.8.18.130:33631 | HEALTHY | <none> | 3 | 0
747982f4b834445b93f077dd1295b2ff | 127.8.18.129:40911 | HEALTHY | <none> | 0 | 0
d06d8f442c824758ae944ffe71e3cb44 | 127.8.18.131:36507 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.8.18.131:36507: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+----------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.8.18.130:33631, 127.8.18.129:40911
ipki_server_key_size | 768 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.129 | experimental | 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.130 | experimental | 127.8.18.130:33631
never_fsync | true | unsafe,advanced | 127.8.18.130:33631, 127.8.18.129:40911
openssl_security_level_override | 0 | unsafe,hidden | 127.8.18.130:33631, 127.8.18.129:40911
rpc_server_allow_ephemeral_ports | true | unsafe | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_format | pb | hidden | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-0/data/info.pb | hidden | 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-1/data/info.pb | hidden | 127.8.18.130:33631
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+--------------------+----------------------------------------
builtin_ntp_servers | 127.8.18.148:42749 | 127.8.18.130:33631, 127.8.18.129:40911
time_source | builtin | 127.8.18.130:33631, 127.8.18.129:40911
Version Summary
Version | Servers
-----------------+-----------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.8.18.190:40323, tserver@127.8.18.129:40911, tserver@127.8.18.130:33631
Tablet Summary
Tablet 1f99182d93ed483e9f520bf312a591e3 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
All reported replicas are:
A = d06d8f442c824758ae944ffe71e3cb44
B = 01cff8a283e94658aea5fad9fa0b2db4
C = 747982f4b834445b93f077dd1295b2ff
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | -1 | Yes
C | A B* C | 1 | -1 | Yes
Tablet 459e1d55c65e4857a7ccba6b13316243 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet b29f930721b943039d1df1f03e073a74 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | -1 | Yes
B | A B* C | 2 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:30:17.424363 8266 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:30:17.881582 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 201: this message will repeat every 5th retry.
I20260810 22:30:18.476151 8670 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20260810 22:30:18.476158 8486 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+--------------------+---------
e52d1e743df1434f88d46b1016be2ae7 | 127.8.18.190:40323 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.8.18.148:42749 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+--------------------+-------------+----------+----------------+-----------------
01cff8a283e94658aea5fad9fa0b2db4 | 127.8.18.130:33631 | HEALTHY | <none> | 3 | 0
747982f4b834445b93f077dd1295b2ff | 127.8.18.129:40911 | HEALTHY | <none> | 0 | 0
d06d8f442c824758ae944ffe71e3cb44 | 127.8.18.131:36507 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.8.18.131:36507: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+----------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.8.18.130:33631, 127.8.18.129:40911
ipki_server_key_size | 768 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.129 | experimental | 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.130 | experimental | 127.8.18.130:33631
never_fsync | true | unsafe,advanced | 127.8.18.130:33631, 127.8.18.129:40911
openssl_security_level_override | 0 | unsafe,hidden | 127.8.18.130:33631, 127.8.18.129:40911
rpc_server_allow_ephemeral_ports | true | unsafe | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_format | pb | hidden | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-0/data/info.pb | hidden | 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-1/data/info.pb | hidden | 127.8.18.130:33631
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+--------------------+----------------------------------------
builtin_ntp_servers | 127.8.18.148:42749 | 127.8.18.130:33631, 127.8.18.129:40911
time_source | builtin | 127.8.18.130:33631, 127.8.18.129:40911
Version Summary
Version | Servers
-----------------+-----------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.8.18.190:40323, tserver@127.8.18.130:33631, tserver@127.8.18.129:40911
Tablet Summary
Tablet 1f99182d93ed483e9f520bf312a591e3 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
All reported replicas are:
A = d06d8f442c824758ae944ffe71e3cb44
B = 01cff8a283e94658aea5fad9fa0b2db4
C = 747982f4b834445b93f077dd1295b2ff
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | -1 | Yes
C | A B* C | 1 | -1 | Yes
Tablet 459e1d55c65e4857a7ccba6b13316243 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet b29f930721b943039d1df1f03e073a74 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | -1 | Yes
B | A B* C | 2 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:30:18.494625 8266 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
I20260810 22:30:19.548782 8670 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20260810 22:30:19.549127 8486 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+--------------------+---------
e52d1e743df1434f88d46b1016be2ae7 | 127.8.18.190:40323 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.8.18.148:42749 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+--------------------+-------------+----------+----------------+-----------------
01cff8a283e94658aea5fad9fa0b2db4 | 127.8.18.130:33631 | HEALTHY | <none> | 3 | 0
747982f4b834445b93f077dd1295b2ff | 127.8.18.129:40911 | HEALTHY | <none> | 0 | 0
d06d8f442c824758ae944ffe71e3cb44 | 127.8.18.131:36507 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.8.18.131:36507: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+----------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.8.18.130:33631, 127.8.18.129:40911
ipki_server_key_size | 768 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.129 | experimental | 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.130 | experimental | 127.8.18.130:33631
never_fsync | true | unsafe,advanced | 127.8.18.130:33631, 127.8.18.129:40911
openssl_security_level_override | 0 | unsafe,hidden | 127.8.18.130:33631, 127.8.18.129:40911
rpc_server_allow_ephemeral_ports | true | unsafe | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_format | pb | hidden | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-0/data/info.pb | hidden | 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-1/data/info.pb | hidden | 127.8.18.130:33631
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+--------------------+----------------------------------------
builtin_ntp_servers | 127.8.18.148:42749 | 127.8.18.130:33631, 127.8.18.129:40911
time_source | builtin | 127.8.18.130:33631, 127.8.18.129:40911
Version Summary
Version | Servers
-----------------+-----------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.8.18.190:40323, tserver@127.8.18.130:33631, tserver@127.8.18.129:40911
Tablet Summary
Tablet 1f99182d93ed483e9f520bf312a591e3 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
All reported replicas are:
A = d06d8f442c824758ae944ffe71e3cb44
B = 01cff8a283e94658aea5fad9fa0b2db4
C = 747982f4b834445b93f077dd1295b2ff
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | -1 | Yes
C | A B* C | 1 | -1 | Yes
Tablet 459e1d55c65e4857a7ccba6b13316243 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet b29f930721b943039d1df1f03e073a74 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | -1 | Yes
B | A B* C | 2 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:30:19.571717 8266 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:30:19.734252 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 201: this message will repeat every 5th retry.
W20260810 22:30:19.872233 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 201: this message will repeat every 5th retry.
W20260810 22:30:20.388772 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 206: this message will repeat every 5th retry.
I20260810 22:30:20.634676 8670 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20260810 22:30:20.635043 8486 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+--------------------+---------
e52d1e743df1434f88d46b1016be2ae7 | 127.8.18.190:40323 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.8.18.148:42749 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+--------------------+-------------+----------+----------------+-----------------
01cff8a283e94658aea5fad9fa0b2db4 | 127.8.18.130:33631 | HEALTHY | <none> | 3 | 0
747982f4b834445b93f077dd1295b2ff | 127.8.18.129:40911 | HEALTHY | <none> | 0 | 0
d06d8f442c824758ae944ffe71e3cb44 | 127.8.18.131:36507 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.8.18.131:36507: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+----------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.8.18.130:33631, 127.8.18.129:40911
ipki_server_key_size | 768 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.129 | experimental | 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.130 | experimental | 127.8.18.130:33631
never_fsync | true | unsafe,advanced | 127.8.18.130:33631, 127.8.18.129:40911
openssl_security_level_override | 0 | unsafe,hidden | 127.8.18.130:33631, 127.8.18.129:40911
rpc_server_allow_ephemeral_ports | true | unsafe | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_format | pb | hidden | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-0/data/info.pb | hidden | 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-1/data/info.pb | hidden | 127.8.18.130:33631
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+--------------------+----------------------------------------
builtin_ntp_servers | 127.8.18.148:42749 | 127.8.18.130:33631, 127.8.18.129:40911
time_source | builtin | 127.8.18.130:33631, 127.8.18.129:40911
Version Summary
Version | Servers
-----------------+-----------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.8.18.190:40323, tserver@127.8.18.130:33631, tserver@127.8.18.129:40911
Tablet Summary
Tablet 1f99182d93ed483e9f520bf312a591e3 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
All reported replicas are:
A = d06d8f442c824758ae944ffe71e3cb44
B = 01cff8a283e94658aea5fad9fa0b2db4
C = 747982f4b834445b93f077dd1295b2ff
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | -1 | Yes
C | A B* C | 1 | -1 | Yes
Tablet 459e1d55c65e4857a7ccba6b13316243 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet b29f930721b943039d1df1f03e073a74 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | -1 | Yes
B | A B* C | 2 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:30:20.659931 8266 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:30:21.331691 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 29 similar messages]
I20260810 22:30:21.714610 8486 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20260810 22:30:21.715183 8670 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+--------------------+---------
e52d1e743df1434f88d46b1016be2ae7 | 127.8.18.190:40323 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.8.18.148:42749 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+--------------------+-------------+----------+----------------+-----------------
01cff8a283e94658aea5fad9fa0b2db4 | 127.8.18.130:33631 | HEALTHY | <none> | 3 | 0
747982f4b834445b93f077dd1295b2ff | 127.8.18.129:40911 | HEALTHY | <none> | 0 | 0
d06d8f442c824758ae944ffe71e3cb44 | 127.8.18.131:36507 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.8.18.131:36507: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+----------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.8.18.130:33631, 127.8.18.129:40911
ipki_server_key_size | 768 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.129 | experimental | 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.130 | experimental | 127.8.18.130:33631
never_fsync | true | unsafe,advanced | 127.8.18.130:33631, 127.8.18.129:40911
openssl_security_level_override | 0 | unsafe,hidden | 127.8.18.130:33631, 127.8.18.129:40911
rpc_server_allow_ephemeral_ports | true | unsafe | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_format | pb | hidden | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-0/data/info.pb | hidden | 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-1/data/info.pb | hidden | 127.8.18.130:33631
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+--------------------+----------------------------------------
builtin_ntp_servers | 127.8.18.148:42749 | 127.8.18.130:33631, 127.8.18.129:40911
time_source | builtin | 127.8.18.130:33631, 127.8.18.129:40911
Version Summary
Version | Servers
-----------------+-----------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.8.18.190:40323, tserver@127.8.18.129:40911, tserver@127.8.18.130:33631
Tablet Summary
Tablet 1f99182d93ed483e9f520bf312a591e3 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
All reported replicas are:
A = d06d8f442c824758ae944ffe71e3cb44
B = 01cff8a283e94658aea5fad9fa0b2db4
C = 747982f4b834445b93f077dd1295b2ff
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | -1 | Yes
C | A B* C | 1 | -1 | Yes
Tablet 459e1d55c65e4857a7ccba6b13316243 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet b29f930721b943039d1df1f03e073a74 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | -1 | Yes
B | A B* C | 2 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:30:21.739449 8266 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:30:22.099550 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 206: this message will repeat every 5th retry.
W20260810 22:30:22.412545 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 206: this message will repeat every 5th retry.
I20260810 22:30:22.798255 8670 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20260810 22:30:22.798537 8486 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+--------------------+---------
e52d1e743df1434f88d46b1016be2ae7 | 127.8.18.190:40323 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.8.18.148:42749 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+--------------------+-------------+----------+----------------+-----------------
01cff8a283e94658aea5fad9fa0b2db4 | 127.8.18.130:33631 | HEALTHY | <none> | 3 | 0
747982f4b834445b93f077dd1295b2ff | 127.8.18.129:40911 | HEALTHY | <none> | 0 | 0
d06d8f442c824758ae944ffe71e3cb44 | 127.8.18.131:36507 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.8.18.131:36507: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+----------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.8.18.130:33631, 127.8.18.129:40911
ipki_server_key_size | 768 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.129 | experimental | 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.130 | experimental | 127.8.18.130:33631
never_fsync | true | unsafe,advanced | 127.8.18.130:33631, 127.8.18.129:40911
openssl_security_level_override | 0 | unsafe,hidden | 127.8.18.130:33631, 127.8.18.129:40911
rpc_server_allow_ephemeral_ports | true | unsafe | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_format | pb | hidden | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-0/data/info.pb | hidden | 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-1/data/info.pb | hidden | 127.8.18.130:33631
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+--------------------+----------------------------------------
builtin_ntp_servers | 127.8.18.148:42749 | 127.8.18.130:33631, 127.8.18.129:40911
time_source | builtin | 127.8.18.130:33631, 127.8.18.129:40911
Version Summary
Version | Servers
-----------------+-----------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.8.18.190:40323, tserver@127.8.18.130:33631, tserver@127.8.18.129:40911
Tablet Summary
Tablet 1f99182d93ed483e9f520bf312a591e3 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
All reported replicas are:
A = d06d8f442c824758ae944ffe71e3cb44
B = 01cff8a283e94658aea5fad9fa0b2db4
C = 747982f4b834445b93f077dd1295b2ff
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | -1 | Yes
C | A B* C | 1 | -1 | Yes
Tablet 459e1d55c65e4857a7ccba6b13316243 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet b29f930721b943039d1df1f03e073a74 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | -1 | Yes
B | A B* C | 2 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:30:22.818423 8266 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:30:22.868999 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 211: this message will repeat every 5th retry.
I20260810 22:30:23.872975 8486 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20260810 22:30:23.873208 8670 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+--------------------+---------
e52d1e743df1434f88d46b1016be2ae7 | 127.8.18.190:40323 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.8.18.148:42749 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+--------------------+-------------+----------+----------------+-----------------
01cff8a283e94658aea5fad9fa0b2db4 | 127.8.18.130:33631 | HEALTHY | <none> | 3 | 0
747982f4b834445b93f077dd1295b2ff | 127.8.18.129:40911 | HEALTHY | <none> | 0 | 0
d06d8f442c824758ae944ffe71e3cb44 | 127.8.18.131:36507 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.8.18.131:36507: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+----------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.8.18.130:33631, 127.8.18.129:40911
ipki_server_key_size | 768 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.129 | experimental | 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.130 | experimental | 127.8.18.130:33631
never_fsync | true | unsafe,advanced | 127.8.18.130:33631, 127.8.18.129:40911
openssl_security_level_override | 0 | unsafe,hidden | 127.8.18.130:33631, 127.8.18.129:40911
rpc_server_allow_ephemeral_ports | true | unsafe | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_format | pb | hidden | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-0/data/info.pb | hidden | 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-1/data/info.pb | hidden | 127.8.18.130:33631
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+--------------------+----------------------------------------
builtin_ntp_servers | 127.8.18.148:42749 | 127.8.18.130:33631, 127.8.18.129:40911
time_source | builtin | 127.8.18.130:33631, 127.8.18.129:40911
Version Summary
Version | Servers
-----------------+-----------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.8.18.190:40323, tserver@127.8.18.129:40911, tserver@127.8.18.130:33631
Tablet Summary
Tablet 1f99182d93ed483e9f520bf312a591e3 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
All reported replicas are:
A = d06d8f442c824758ae944ffe71e3cb44
B = 01cff8a283e94658aea5fad9fa0b2db4
C = 747982f4b834445b93f077dd1295b2ff
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | -1 | Yes
C | A B* C | 1 | -1 | Yes
Tablet 459e1d55c65e4857a7ccba6b13316243 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet b29f930721b943039d1df1f03e073a74 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | -1 | Yes
B | A B* C | 2 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:30:23.893071 8266 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:30:24.879415 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 211: this message will repeat every 5th retry.
I20260810 22:30:24.948107 8670 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20260810 22:30:24.948606 8486 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+--------------------+---------
e52d1e743df1434f88d46b1016be2ae7 | 127.8.18.190:40323 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.8.18.148:42749 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+--------------------+-------------+----------+----------------+-----------------
01cff8a283e94658aea5fad9fa0b2db4 | 127.8.18.130:33631 | HEALTHY | <none> | 3 | 0
747982f4b834445b93f077dd1295b2ff | 127.8.18.129:40911 | HEALTHY | <none> | 0 | 0
d06d8f442c824758ae944ffe71e3cb44 | 127.8.18.131:36507 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.8.18.131:36507: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+----------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.8.18.130:33631, 127.8.18.129:40911
ipki_server_key_size | 768 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.129 | experimental | 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.130 | experimental | 127.8.18.130:33631
never_fsync | true | unsafe,advanced | 127.8.18.130:33631, 127.8.18.129:40911
openssl_security_level_override | 0 | unsafe,hidden | 127.8.18.130:33631, 127.8.18.129:40911
rpc_server_allow_ephemeral_ports | true | unsafe | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_format | pb | hidden | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-0/data/info.pb | hidden | 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-1/data/info.pb | hidden | 127.8.18.130:33631
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+--------------------+----------------------------------------
builtin_ntp_servers | 127.8.18.148:42749 | 127.8.18.130:33631, 127.8.18.129:40911
time_source | builtin | 127.8.18.130:33631, 127.8.18.129:40911
Version Summary
Version | Servers
-----------------+-----------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.8.18.190:40323, tserver@127.8.18.130:33631, tserver@127.8.18.129:40911
Tablet Summary
Tablet 1f99182d93ed483e9f520bf312a591e3 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
All reported replicas are:
A = d06d8f442c824758ae944ffe71e3cb44
B = 01cff8a283e94658aea5fad9fa0b2db4
C = 747982f4b834445b93f077dd1295b2ff
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | -1 | Yes
C | A B* C | 1 | -1 | Yes
Tablet 459e1d55c65e4857a7ccba6b13316243 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet b29f930721b943039d1df1f03e073a74 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | -1 | Yes
B | A B* C | 2 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:30:24.973611 8266 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:30:25.077042 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 211: this message will repeat every 5th retry.
W20260810 22:30:25.550972 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 216: this message will repeat every 5th retry.
I20260810 22:30:26.025969 8670 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20260810 22:30:26.026229 8486 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+--------------------+---------
e52d1e743df1434f88d46b1016be2ae7 | 127.8.18.190:40323 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.8.18.148:42749 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+--------------------+-------------+----------+----------------+-----------------
01cff8a283e94658aea5fad9fa0b2db4 | 127.8.18.130:33631 | HEALTHY | <none> | 3 | 0
747982f4b834445b93f077dd1295b2ff | 127.8.18.129:40911 | HEALTHY | <none> | 0 | 0
d06d8f442c824758ae944ffe71e3cb44 | 127.8.18.131:36507 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.8.18.131:36507: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+----------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.8.18.130:33631, 127.8.18.129:40911
ipki_server_key_size | 768 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.129 | experimental | 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.130 | experimental | 127.8.18.130:33631
never_fsync | true | unsafe,advanced | 127.8.18.130:33631, 127.8.18.129:40911
openssl_security_level_override | 0 | unsafe,hidden | 127.8.18.130:33631, 127.8.18.129:40911
rpc_server_allow_ephemeral_ports | true | unsafe | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_format | pb | hidden | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-0/data/info.pb | hidden | 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-1/data/info.pb | hidden | 127.8.18.130:33631
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+--------------------+----------------------------------------
builtin_ntp_servers | 127.8.18.148:42749 | 127.8.18.130:33631, 127.8.18.129:40911
time_source | builtin | 127.8.18.130:33631, 127.8.18.129:40911
Version Summary
Version | Servers
-----------------+-----------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.8.18.190:40323, tserver@127.8.18.130:33631, tserver@127.8.18.129:40911
Tablet Summary
Tablet 1f99182d93ed483e9f520bf312a591e3 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
All reported replicas are:
A = d06d8f442c824758ae944ffe71e3cb44
B = 01cff8a283e94658aea5fad9fa0b2db4
C = 747982f4b834445b93f077dd1295b2ff
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | -1 | Yes
C | A B* C | 1 | -1 | Yes
Tablet 459e1d55c65e4857a7ccba6b13316243 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet b29f930721b943039d1df1f03e073a74 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | -1 | Yes
B | A B* C | 2 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:30:26.046950 8266 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:30:26.548906 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 29 similar messages]
I20260810 22:30:27.105857 8670 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20260810 22:30:27.107292 8486 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+--------------------+---------
e52d1e743df1434f88d46b1016be2ae7 | 127.8.18.190:40323 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.8.18.148:42749 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+--------------------+-------------+----------+----------------+-----------------
01cff8a283e94658aea5fad9fa0b2db4 | 127.8.18.130:33631 | HEALTHY | <none> | 3 | 0
747982f4b834445b93f077dd1295b2ff | 127.8.18.129:40911 | HEALTHY | <none> | 0 | 0
d06d8f442c824758ae944ffe71e3cb44 | 127.8.18.131:36507 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.8.18.131:36507: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+----------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.8.18.130:33631, 127.8.18.129:40911
ipki_server_key_size | 768 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.129 | experimental | 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.130 | experimental | 127.8.18.130:33631
never_fsync | true | unsafe,advanced | 127.8.18.130:33631, 127.8.18.129:40911
openssl_security_level_override | 0 | unsafe,hidden | 127.8.18.130:33631, 127.8.18.129:40911
rpc_server_allow_ephemeral_ports | true | unsafe | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_format | pb | hidden | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-0/data/info.pb | hidden | 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-1/data/info.pb | hidden | 127.8.18.130:33631
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+--------------------+----------------------------------------
builtin_ntp_servers | 127.8.18.148:42749 | 127.8.18.130:33631, 127.8.18.129:40911
time_source | builtin | 127.8.18.130:33631, 127.8.18.129:40911
Version Summary
Version | Servers
-----------------+-----------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.8.18.190:40323, tserver@127.8.18.130:33631, tserver@127.8.18.129:40911
Tablet Summary
Tablet 1f99182d93ed483e9f520bf312a591e3 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
All reported replicas are:
A = d06d8f442c824758ae944ffe71e3cb44
B = 01cff8a283e94658aea5fad9fa0b2db4
C = 747982f4b834445b93f077dd1295b2ff
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | -1 | Yes
C | A B* C | 1 | -1 | Yes
Tablet 459e1d55c65e4857a7ccba6b13316243 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet b29f930721b943039d1df1f03e073a74 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | -1 | Yes
B | A B* C | 2 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:30:27.130913 8266 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:30:27.455700 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 216: this message will repeat every 5th retry.
W20260810 22:30:27.639148 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 216: this message will repeat every 5th retry.
W20260810 22:30:28.130445 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 221: this message will repeat every 5th retry.
I20260810 22:30:28.190029 8670 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20260810 22:30:28.190281 8486 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+--------------------+---------
e52d1e743df1434f88d46b1016be2ae7 | 127.8.18.190:40323 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.8.18.148:42749 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+--------------------+-------------+----------+----------------+-----------------
01cff8a283e94658aea5fad9fa0b2db4 | 127.8.18.130:33631 | HEALTHY | <none> | 3 | 0
747982f4b834445b93f077dd1295b2ff | 127.8.18.129:40911 | HEALTHY | <none> | 0 | 0
d06d8f442c824758ae944ffe71e3cb44 | 127.8.18.131:36507 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.8.18.131:36507: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+----------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.8.18.130:33631, 127.8.18.129:40911
ipki_server_key_size | 768 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.129 | experimental | 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.130 | experimental | 127.8.18.130:33631
never_fsync | true | unsafe,advanced | 127.8.18.130:33631, 127.8.18.129:40911
openssl_security_level_override | 0 | unsafe,hidden | 127.8.18.130:33631, 127.8.18.129:40911
rpc_server_allow_ephemeral_ports | true | unsafe | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_format | pb | hidden | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-0/data/info.pb | hidden | 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-1/data/info.pb | hidden | 127.8.18.130:33631
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+--------------------+----------------------------------------
builtin_ntp_servers | 127.8.18.148:42749 | 127.8.18.130:33631, 127.8.18.129:40911
time_source | builtin | 127.8.18.130:33631, 127.8.18.129:40911
Version Summary
Version | Servers
-----------------+-----------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.8.18.190:40323, tserver@127.8.18.130:33631, tserver@127.8.18.129:40911
Tablet Summary
Tablet 1f99182d93ed483e9f520bf312a591e3 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
All reported replicas are:
A = d06d8f442c824758ae944ffe71e3cb44
B = 01cff8a283e94658aea5fad9fa0b2db4
C = 747982f4b834445b93f077dd1295b2ff
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | -1 | Yes
C | A B* C | 1 | -1 | Yes
Tablet 459e1d55c65e4857a7ccba6b13316243 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet b29f930721b943039d1df1f03e073a74 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | -1 | Yes
B | A B* C | 2 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:30:28.210738 8266 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
I20260810 22:30:29.266749 8486 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20260810 22:30:29.266819 8670 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+--------------------+---------
e52d1e743df1434f88d46b1016be2ae7 | 127.8.18.190:40323 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.8.18.148:42749 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+--------------------+-------------+----------+----------------+-----------------
01cff8a283e94658aea5fad9fa0b2db4 | 127.8.18.130:33631 | HEALTHY | <none> | 3 | 0
747982f4b834445b93f077dd1295b2ff | 127.8.18.129:40911 | HEALTHY | <none> | 0 | 0
d06d8f442c824758ae944ffe71e3cb44 | 127.8.18.131:36507 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.8.18.131:36507: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+----------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.8.18.130:33631, 127.8.18.129:40911
ipki_server_key_size | 768 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.129 | experimental | 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.130 | experimental | 127.8.18.130:33631
never_fsync | true | unsafe,advanced | 127.8.18.130:33631, 127.8.18.129:40911
openssl_security_level_override | 0 | unsafe,hidden | 127.8.18.130:33631, 127.8.18.129:40911
rpc_server_allow_ephemeral_ports | true | unsafe | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_format | pb | hidden | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-0/data/info.pb | hidden | 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-1/data/info.pb | hidden | 127.8.18.130:33631
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+--------------------+----------------------------------------
builtin_ntp_servers | 127.8.18.148:42749 | 127.8.18.130:33631, 127.8.18.129:40911
time_source | builtin | 127.8.18.130:33631, 127.8.18.129:40911
Version Summary
Version | Servers
-----------------+-----------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.8.18.190:40323, tserver@127.8.18.129:40911, tserver@127.8.18.130:33631
Tablet Summary
Tablet 1f99182d93ed483e9f520bf312a591e3 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
All reported replicas are:
A = d06d8f442c824758ae944ffe71e3cb44
B = 01cff8a283e94658aea5fad9fa0b2db4
C = 747982f4b834445b93f077dd1295b2ff
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | -1 | Yes
C | A B* C | 1 | -1 | Yes
Tablet 459e1d55c65e4857a7ccba6b13316243 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet b29f930721b943039d1df1f03e073a74 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | -1 | Yes
B | A B* C | 2 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:30:29.287380 8266 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:30:29.615064 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 221: this message will repeat every 5th retry.
W20260810 22:30:30.314248 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 221: this message will repeat every 5th retry.
I20260810 22:30:30.342257 8486 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20260810 22:30:30.342622 8670 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+--------------------+---------
e52d1e743df1434f88d46b1016be2ae7 | 127.8.18.190:40323 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.8.18.148:42749 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+--------------------+-------------+----------+----------------+-----------------
01cff8a283e94658aea5fad9fa0b2db4 | 127.8.18.130:33631 | HEALTHY | <none> | 3 | 0
747982f4b834445b93f077dd1295b2ff | 127.8.18.129:40911 | HEALTHY | <none> | 0 | 0
d06d8f442c824758ae944ffe71e3cb44 | 127.8.18.131:36507 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.8.18.131:36507: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+----------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.8.18.130:33631, 127.8.18.129:40911
ipki_server_key_size | 768 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.129 | experimental | 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.130 | experimental | 127.8.18.130:33631
never_fsync | true | unsafe,advanced | 127.8.18.130:33631, 127.8.18.129:40911
openssl_security_level_override | 0 | unsafe,hidden | 127.8.18.130:33631, 127.8.18.129:40911
rpc_server_allow_ephemeral_ports | true | unsafe | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_format | pb | hidden | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-0/data/info.pb | hidden | 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-1/data/info.pb | hidden | 127.8.18.130:33631
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+--------------------+----------------------------------------
builtin_ntp_servers | 127.8.18.148:42749 | 127.8.18.130:33631, 127.8.18.129:40911
time_source | builtin | 127.8.18.130:33631, 127.8.18.129:40911
Version Summary
Version | Servers
-----------------+-----------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.8.18.190:40323, tserver@127.8.18.129:40911, tserver@127.8.18.130:33631
Tablet Summary
Tablet 1f99182d93ed483e9f520bf312a591e3 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
All reported replicas are:
A = d06d8f442c824758ae944ffe71e3cb44
B = 01cff8a283e94658aea5fad9fa0b2db4
C = 747982f4b834445b93f077dd1295b2ff
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | -1 | Yes
C | A B* C | 1 | -1 | Yes
Tablet 459e1d55c65e4857a7ccba6b13316243 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet b29f930721b943039d1df1f03e073a74 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | -1 | Yes
B | A B* C | 2 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:30:30.362103 8266 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:30:30.780295 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 226: this message will repeat every 5th retry.
I20260810 22:30:31.415302 8670 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20260810 22:30:31.415607 8486 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+--------------------+---------
e52d1e743df1434f88d46b1016be2ae7 | 127.8.18.190:40323 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.8.18.148:42749 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+--------------------+-------------+----------+----------------+-----------------
01cff8a283e94658aea5fad9fa0b2db4 | 127.8.18.130:33631 | HEALTHY | <none> | 3 | 0
747982f4b834445b93f077dd1295b2ff | 127.8.18.129:40911 | HEALTHY | <none> | 0 | 0
d06d8f442c824758ae944ffe71e3cb44 | 127.8.18.131:36507 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.8.18.131:36507: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+----------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.8.18.130:33631, 127.8.18.129:40911
ipki_server_key_size | 768 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.129 | experimental | 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.130 | experimental | 127.8.18.130:33631
never_fsync | true | unsafe,advanced | 127.8.18.130:33631, 127.8.18.129:40911
openssl_security_level_override | 0 | unsafe,hidden | 127.8.18.130:33631, 127.8.18.129:40911
rpc_server_allow_ephemeral_ports | true | unsafe | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_format | pb | hidden | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-0/data/info.pb | hidden | 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-1/data/info.pb | hidden | 127.8.18.130:33631
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+--------------------+----------------------------------------
builtin_ntp_servers | 127.8.18.148:42749 | 127.8.18.130:33631, 127.8.18.129:40911
time_source | builtin | 127.8.18.130:33631, 127.8.18.129:40911
Version Summary
Version | Servers
-----------------+-----------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.8.18.190:40323, tserver@127.8.18.130:33631, tserver@127.8.18.129:40911
Tablet Summary
Tablet 1f99182d93ed483e9f520bf312a591e3 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
All reported replicas are:
A = d06d8f442c824758ae944ffe71e3cb44
B = 01cff8a283e94658aea5fad9fa0b2db4
C = 747982f4b834445b93f077dd1295b2ff
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | -1 | Yes
C | A B* C | 1 | -1 | Yes
Tablet 459e1d55c65e4857a7ccba6b13316243 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet b29f930721b943039d1df1f03e073a74 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | -1 | Yes
B | A B* C | 2 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:30:31.436107 8266 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:30:31.736440 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 30 similar messages]
W20260810 22:30:32.215785 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 226: this message will repeat every 5th retry.
I20260810 22:30:32.489485 8670 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20260810 22:30:32.489485 8486 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+--------------------+---------
e52d1e743df1434f88d46b1016be2ae7 | 127.8.18.190:40323 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.8.18.148:42749 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+--------------------+-------------+----------+----------------+-----------------
01cff8a283e94658aea5fad9fa0b2db4 | 127.8.18.130:33631 | HEALTHY | <none> | 3 | 0
747982f4b834445b93f077dd1295b2ff | 127.8.18.129:40911 | HEALTHY | <none> | 0 | 0
d06d8f442c824758ae944ffe71e3cb44 | 127.8.18.131:36507 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.8.18.131:36507: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+----------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.8.18.130:33631, 127.8.18.129:40911
ipki_server_key_size | 768 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.129 | experimental | 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.130 | experimental | 127.8.18.130:33631
never_fsync | true | unsafe,advanced | 127.8.18.130:33631, 127.8.18.129:40911
openssl_security_level_override | 0 | unsafe,hidden | 127.8.18.130:33631, 127.8.18.129:40911
rpc_server_allow_ephemeral_ports | true | unsafe | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_format | pb | hidden | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-0/data/info.pb | hidden | 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-1/data/info.pb | hidden | 127.8.18.130:33631
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+--------------------+----------------------------------------
builtin_ntp_servers | 127.8.18.148:42749 | 127.8.18.130:33631, 127.8.18.129:40911
time_source | builtin | 127.8.18.130:33631, 127.8.18.129:40911
Version Summary
Version | Servers
-----------------+-----------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.8.18.190:40323, tserver@127.8.18.130:33631, tserver@127.8.18.129:40911
Tablet Summary
Tablet 1f99182d93ed483e9f520bf312a591e3 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
All reported replicas are:
A = d06d8f442c824758ae944ffe71e3cb44
B = 01cff8a283e94658aea5fad9fa0b2db4
C = 747982f4b834445b93f077dd1295b2ff
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | -1 | Yes
C | A B* C | 1 | -1 | Yes
Tablet 459e1d55c65e4857a7ccba6b13316243 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet b29f930721b943039d1df1f03e073a74 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | -1 | Yes
B | A B* C | 2 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:30:32.509711 8266 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:30:32.943406 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 226: this message will repeat every 5th retry.
W20260810 22:30:33.344077 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 231: this message will repeat every 5th retry.
I20260810 22:30:33.569557 8670 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20260810 22:30:33.569919 8486 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+--------------------+---------
e52d1e743df1434f88d46b1016be2ae7 | 127.8.18.190:40323 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.8.18.148:42749 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+--------------------+-------------+----------+----------------+-----------------
01cff8a283e94658aea5fad9fa0b2db4 | 127.8.18.130:33631 | HEALTHY | <none> | 3 | 0
747982f4b834445b93f077dd1295b2ff | 127.8.18.129:40911 | HEALTHY | <none> | 0 | 0
d06d8f442c824758ae944ffe71e3cb44 | 127.8.18.131:36507 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.8.18.131:36507: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+----------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.8.18.130:33631, 127.8.18.129:40911
ipki_server_key_size | 768 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.129 | experimental | 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.130 | experimental | 127.8.18.130:33631
never_fsync | true | unsafe,advanced | 127.8.18.130:33631, 127.8.18.129:40911
openssl_security_level_override | 0 | unsafe,hidden | 127.8.18.130:33631, 127.8.18.129:40911
rpc_server_allow_ephemeral_ports | true | unsafe | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_format | pb | hidden | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-0/data/info.pb | hidden | 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-1/data/info.pb | hidden | 127.8.18.130:33631
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+--------------------+----------------------------------------
builtin_ntp_servers | 127.8.18.148:42749 | 127.8.18.130:33631, 127.8.18.129:40911
time_source | builtin | 127.8.18.130:33631, 127.8.18.129:40911
Version Summary
Version | Servers
-----------------+-----------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.8.18.190:40323, tserver@127.8.18.130:33631, tserver@127.8.18.129:40911
Tablet Summary
Tablet 1f99182d93ed483e9f520bf312a591e3 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
All reported replicas are:
A = d06d8f442c824758ae944ffe71e3cb44
B = 01cff8a283e94658aea5fad9fa0b2db4
C = 747982f4b834445b93f077dd1295b2ff
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | -1 | Yes
C | A B* C | 1 | -1 | Yes
Tablet 459e1d55c65e4857a7ccba6b13316243 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet b29f930721b943039d1df1f03e073a74 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | -1 | Yes
B | A B* C | 2 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:30:33.589547 8266 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:30:34.485353 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 231: this message will repeat every 5th retry.
I20260810 22:30:34.645232 8670 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20260810 22:30:34.645903 8486 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+--------------------+---------
e52d1e743df1434f88d46b1016be2ae7 | 127.8.18.190:40323 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.8.18.148:42749 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+--------------------+-------------+----------+----------------+-----------------
01cff8a283e94658aea5fad9fa0b2db4 | 127.8.18.130:33631 | HEALTHY | <none> | 3 | 0
747982f4b834445b93f077dd1295b2ff | 127.8.18.129:40911 | HEALTHY | <none> | 0 | 0
d06d8f442c824758ae944ffe71e3cb44 | 127.8.18.131:36507 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.8.18.131:36507: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+----------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.8.18.130:33631, 127.8.18.129:40911
ipki_server_key_size | 768 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.129 | experimental | 127.8.18.129:40911
local_ip_for_outbound_sockets | 127.8.18.130 | experimental | 127.8.18.130:33631
never_fsync | true | unsafe,advanced | 127.8.18.130:33631, 127.8.18.129:40911
openssl_security_level_override | 0 | unsafe,hidden | 127.8.18.130:33631, 127.8.18.129:40911
rpc_server_allow_ephemeral_ports | true | unsafe | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_format | pb | hidden | 127.8.18.130:33631, 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-0/data/info.pb | hidden | 127.8.18.129:40911
server_dump_info_path | /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/ts-1/data/info.pb | hidden | 127.8.18.130:33631
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.8.18.130:33631, 127.8.18.129:40911
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+--------------------+----------------------------------------
builtin_ntp_servers | 127.8.18.148:42749 | 127.8.18.130:33631, 127.8.18.129:40911
time_source | builtin | 127.8.18.130:33631, 127.8.18.129:40911
Version Summary
Version | Servers
-----------------+-----------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.8.18.190:40323, tserver@127.8.18.130:33631, tserver@127.8.18.129:40911
Tablet Summary
Tablet 1f99182d93ed483e9f520bf312a591e3 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
All reported replicas are:
A = d06d8f442c824758ae944ffe71e3cb44
B = 01cff8a283e94658aea5fad9fa0b2db4
C = 747982f4b834445b93f077dd1295b2ff
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | -1 | Yes
C | A B* C | 1 | -1 | Yes
Tablet 459e1d55c65e4857a7ccba6b13316243 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet b29f930721b943039d1df1f03e073a74 of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
747982f4b834445b93f077dd1295b2ff (127.8.18.129:40911): RUNNING
01cff8a283e94658aea5fad9fa0b2db4 (127.8.18.130:33631): RUNNING [LEADER]
d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): TS unavailable
All reported replicas are:
A = 747982f4b834445b93f077dd1295b2ff
B = 01cff8a283e94658aea5fad9fa0b2db4
C = d06d8f442c824758ae944ffe71e3cb44
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | -1 | Yes
B | A B* C | 2 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:30:34.664605 8266 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
I20260810 22:30:34.848562 8305 maintenance_manager.cc:419] P e52d1e743df1434f88d46b1016be2ae7: Scheduling FlushMRSOp(00000000000000000000000000000000): perf score=0.033370
W20260810 22:30:34.851440 8303 dir_manager.cc:174] Insufficient disk space under path /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/data/data: will retry after 10 seconds: IO error: Insufficient disk space to allocate 0 bytes under path /tmp/dist-test-taskMIzrRq/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786400914303563-8266-0/minicluster-data/master-0/data/data (497651712 bytes available vs 518351011 bytes reserved) (error 28)
W20260810 22:30:34.851964 8303 tablet_replica_mm_ops.cc:214] T 00000000000000000000000000000000 P e52d1e743df1434f88d46b1016be2ae7: failed to flush MRS: IO error: Failed to open DiskRowSet for flush: tablet 00000000000000000000000000000000: unable to open output file for column entry_type INT8 NOT NULL: No directories could be added (1 dirs total, 0 failed, 1 full): IO error: No directories available in 00000000000000000000000000000000's directory group (1 dirs total, 0 failed, 1 full) (error 28) (error 19)
F20260810 22:30:34.852044 8303 tablet_replica_mm_ops.cc:215] Check failed: tablet->HasBeenStopped() Unrecoverable flush failure caused by error: IO error: Failed to open DiskRowSet for flush: tablet 00000000000000000000000000000000: unable to open output file for column entry_type INT8 NOT NULL: No directories could be added (1 dirs total, 0 failed, 1 full): IO error: No directories available in 00000000000000000000000000000000's directory group (1 dirs total, 0 failed, 1 full) (error 28) (error 19)
*** Check failure stack trace: ***
@ 0x7f36acc549ad google::LogMessage::Fail() at ??:0
@ 0x7f36acc5788a google::LogMessage::SendToLog() at ??:0
@ 0x7f36acc544de google::LogMessage::Flush() at ??:0
@ 0x7f36acc55a19 google::LogMessageFatal::~LogMessageFatal() at ??:0
@ 0x7f36b59cd299 kudu::tablet::FlushMRSOp::Perform() at ??:0
@ 0x7f36aee0a674 kudu::MaintenanceManager::LaunchOp() at ??:0
@ 0x7f36aee0773c _ZZN4kudu18MaintenanceManager18RunSchedulerThreadEvENKUlvE_clEv at ??:0
@ 0x7f36aee0b88c _ZNSt17_Function_handlerIFvvEZN4kudu18MaintenanceManager18RunSchedulerThreadEvEUlvE_E9_M_invokeERKSt9_Any_data at ??:0
@ 0x61d838 std::function<>::operator()() at ??:0
@ 0x7f36aeea63d7 kudu::ThreadPool::DispatchThread() at ??:0
@ 0x7f36aeea6c83 _ZZN4kudu10ThreadPool12CreateThreadEvENKUlvE_clEv at ??:0
@ 0x7f36aeea83f2 _ZNSt17_Function_handlerIFvvEZN4kudu10ThreadPool12CreateThreadEvEUlvE_E9_M_invokeERKSt9_Any_data at ??:0
@ 0x61d838 std::function<>::operator()() at ??:0
@ 0x7f36aee97512 kudu::Thread::SuperviseThread() at ??:0
@ 0x7f36b04db6db start_thread at ??:0
@ 0x7f36ab75261f clone at ??:0
W20260810 22:30:35.482465 8756 heartbeater.cc:644] Failed to heartbeat to 127.8.18.190:40323 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.8.18.190:40323: connect: Connection refused (error 111)
W20260810 22:30:35.482769 8579 heartbeater.cc:644] Failed to heartbeat to 127.8.18.190:40323 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.8.18.190:40323: connect: Connection refused (error 111)
W20260810 22:30:35.632167 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 231: this message will repeat every 5th retry.
W20260810 22:30:35.933558 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 236: this message will repeat every 5th retry.
W20260810 22:30:36.852653 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 30 similar messages]
W20260810 22:30:37.185254 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 236: this message will repeat every 5th retry.
W20260810 22:30:38.008749 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 236: this message will repeat every 5th retry.
W20260810 22:30:38.387241 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 241: this message will repeat every 5th retry.
W20260810 22:30:39.538045 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 241: this message will repeat every 5th retry.
W20260810 22:30:40.453775 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 241: this message will repeat every 5th retry.
W20260810 22:30:40.940037 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 246: this message will repeat every 5th retry.
W20260810 22:30:42.105253 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 30 similar messages]
W20260810 22:30:42.220049 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 246: this message will repeat every 5th retry.
W20260810 22:30:43.163725 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 246: this message will repeat every 5th retry.
W20260810 22:30:43.240767 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 251: this message will repeat every 5th retry.
W20260810 22:30:44.811177 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 251: this message will repeat every 5th retry.
W20260810 22:30:45.610810 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 256: this message will repeat every 5th retry.
W20260810 22:30:45.780478 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 251: this message will repeat every 5th retry.
W20260810 22:30:47.108479 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 29 similar messages]
W20260810 22:30:47.476784 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 256: this message will repeat every 5th retry.
W20260810 22:30:48.024366 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 256: this message will repeat every 5th retry.
W20260810 22:30:48.053776 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 261: this message will repeat every 5th retry.
W20260810 22:30:50.066623 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 261: this message will repeat every 5th retry.
W20260810 22:30:50.417181 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 266: this message will repeat every 5th retry.
W20260810 22:30:50.542070 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 261: this message will repeat every 5th retry.
W20260810 22:30:52.507515 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 31 similar messages]
W20260810 22:30:52.512295 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 266: this message will repeat every 5th retry.
W20260810 22:30:52.990779 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 271: this message will repeat every 5th retry.
W20260810 22:30:53.106915 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 266: this message will repeat every 5th retry.
W20260810 22:30:54.955793 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 271: this message will repeat every 5th retry.
W20260810 22:30:55.293015 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 271: this message will repeat every 5th retry.
W20260810 22:30:55.821089 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 276: this message will repeat every 5th retry.
W20260810 22:30:57.426466 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 276: this message will repeat every 5th retry.
W20260810 22:30:57.659291 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 31 similar messages]
W20260810 22:30:57.660122 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 276: this message will repeat every 5th retry.
W20260810 22:30:58.352887 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 281: this message will repeat every 5th retry.
W20260810 22:30:59.680827 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 281: this message will repeat every 5th retry.
W20260810 22:31:00.111186 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 281: this message will repeat every 5th retry.
W20260810 22:31:00.808248 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 286: this message will repeat every 5th retry.
W20260810 22:31:01.917925 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 286: this message will repeat every 5th retry.
W20260810 22:31:02.759330 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 30 similar messages]
W20260810 22:31:02.760107 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 286: this message will repeat every 5th retry.
W20260810 22:31:03.185810 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 291: this message will repeat every 5th retry.
W20260810 22:31:04.754303 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 291: this message will repeat every 5th retry.
W20260810 22:31:05.106750 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 291: this message will repeat every 5th retry.
W20260810 22:31:05.875777 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 296: this message will repeat every 5th retry.
W20260810 22:31:07.183868 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 296: this message will repeat every 5th retry.
W20260810 22:31:07.716171 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 296: this message will repeat every 5th retry.
W20260810 22:31:07.906603 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 30 similar messages]
W20260810 22:31:08.353129 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 301: this message will repeat every 5th retry.
W20260810 22:31:09.494207 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 301: this message will repeat every 5th retry.
W20260810 22:31:10.234601 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 301: this message will repeat every 5th retry.
W20260810 22:31:10.712903 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 306: this message will repeat every 5th retry.
W20260810 22:31:11.720383 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 306: this message will repeat every 5th retry.
W20260810 22:31:12.612265 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 306: this message will repeat every 5th retry.
W20260810 22:31:13.102845 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 31 similar messages]
W20260810 22:31:13.302141 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 311: this message will repeat every 5th retry.
W20260810 22:31:14.136590 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 311: this message will repeat every 5th retry.
W20260810 22:31:14.989656 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 311: this message will repeat every 5th retry.
W20260810 22:31:15.546659 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 316: this message will repeat every 5th retry.
W20260810 22:31:16.633423 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 316: this message will repeat every 5th retry.
W20260810 22:31:17.476595 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 316: this message will repeat every 5th retry.
W20260810 22:31:17.997148 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 321: this message will repeat every 5th retry.
W20260810 22:31:18.116528 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 31 similar messages]
W20260810 22:31:18.990235 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 321: this message will repeat every 5th retry.
W20260810 22:31:20.035141 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 321: this message will repeat every 5th retry.
W20260810 22:31:20.486369 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 326: this message will repeat every 5th retry.
W20260810 22:31:21.489981 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 326: this message will repeat every 5th retry.
W20260810 22:31:22.503695 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 326: this message will repeat every 5th retry.
W20260810 22:31:22.505080 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 331: this message will repeat every 5th retry.
W20260810 22:31:23.369925 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 31 similar messages]
W20260810 22:31:24.076655 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 331: this message will repeat every 5th retry.
W20260810 22:31:24.888928 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 336: this message will repeat every 5th retry.
W20260810 22:31:24.913642 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 331: this message will repeat every 5th retry.
W20260810 22:31:26.723930 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 336: this message will repeat every 5th retry.
W20260810 22:31:27.369326 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 341: this message will repeat every 5th retry.
W20260810 22:31:27.474000 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 336: this message will repeat every 5th retry.
W20260810 22:31:28.403257 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 29 similar messages]
W20260810 22:31:29.338797 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 341: this message will repeat every 5th retry.
W20260810 22:31:29.713454 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 341: this message will repeat every 5th retry.
W20260810 22:31:29.954295 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 346: this message will repeat every 5th retry.
W20260810 22:31:31.970198 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 346: this message will repeat every 5th retry.
W20260810 22:31:32.091105 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 346: this message will repeat every 5th retry.
W20260810 22:31:32.287201 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 351: this message will repeat every 5th retry.
W20260810 22:31:33.504510 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 30 similar messages]
W20260810 22:31:34.527815 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 351: this message will repeat every 5th retry.
W20260810 22:31:34.586519 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 351: this message will repeat every 5th retry.
W20260810 22:31:34.721909 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 356: this message will repeat every 5th retry.
W20260810 22:31:35.556121 8756 heartbeater.cc:644] Failed to heartbeat to 127.8.18.190:40323 (60 consecutive failures): Network error: Failed to ping master at 127.8.18.190:40323: Client connection negotiation failed: client connection to 127.8.18.190:40323: connect: Connection refused (error 111)
W20260810 22:31:35.556495 8579 heartbeater.cc:644] Failed to heartbeat to 127.8.18.190:40323 (60 consecutive failures): Network error: Failed to ping master at 127.8.18.190:40323: Client connection negotiation failed: client connection to 127.8.18.190:40323: connect: Connection refused (error 111)
Master Summary
UUID | Address | Status
--------------------------------+--------------------+-------------
<unknown> (127.8.18.190:40323) | 127.8.18.190:40323 | UNAVAILABLE
Error from 127.8.18.190:40323: Network error: Client connection negotiation failed: client connection to 127.8.18.190:40323: connect: Connection refused (error 111) (UNAVAILABLE)
All reported replicas are:
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+----------+--------------+--------------+------------
Tablet Server Summary
Version Summary
Version | Servers
---------+---------
The cluster doesn't have any matching tablets
The cluster doesn't have any matching system tables
The cluster doesn't have any matching tables
==================
Warnings:
==================
master unusual flags check error: 1 of 1 masters were not available to retrieve unusual flags
master diverged flags check error: 1 of 1 masters were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from masters: failed to gather info from all masters: 1 of 1 had errors
Not found: master consensus error: no master consensus state available
Timed out: leader master liveness check error: Could not connect to the cluster: ConnectToClusterRpc(addrs: 127.8.18.190:40323, num_attempts: 344) passed its deadline: Network error: Client connection negotiation failed: client connection to 127.8.18.190:40323: connect: Connection refused (error 111)
FAILED
I20260810 22:31:35.859067 8266 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Timed out: leader master liveness check error: Could not connect to the cluster: ConnectToClusterRpc(addrs: 127.8.18.190:40323, num_attempts: 344) passed its deadline: Network error: Client connection negotiation failed: client connection to 127.8.18.190:40323: connect: Connection refused (error 111)
W20260810 22:31:36.771239 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 356: this message will repeat every 5th retry.
W20260810 22:31:37.057574 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 356: this message will repeat every 5th retry.
W20260810 22:31:37.063256 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 361: this message will repeat every 5th retry.
W20260810 22:31:38.575351 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 30 similar messages]
W20260810 22:31:39.585987 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 361: this message will repeat every 5th retry.
W20260810 22:31:39.625607 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 366: this message will repeat every 5th retry.
W20260810 22:31:39.839327 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 361: this message will repeat every 5th retry.
W20260810 22:31:42.184108 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 366: this message will repeat every 5th retry.
W20260810 22:31:42.190165 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 366: this message will repeat every 5th retry.
W20260810 22:31:42.329959 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 371: this message will repeat every 5th retry.
W20260810 22:31:43.689214 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 28 similar messages]
W20260810 22:31:44.643067 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 371: this message will repeat every 5th retry.
W20260810 22:31:44.841887 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 371: this message will repeat every 5th retry.
W20260810 22:31:44.856611 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 376: this message will repeat every 5th retry.
W20260810 22:31:47.071979 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 376: this message will repeat every 5th retry.
W20260810 22:31:47.466259 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 381: this message will repeat every 5th retry.
W20260810 22:31:47.489012 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 376: this message will repeat every 5th retry.
W20260810 22:31:48.834309 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 30 similar messages]
W20260810 22:31:49.453663 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 381: this message will repeat every 5th retry.
W20260810 22:31:49.650884 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 386: this message will repeat every 5th retry.
W20260810 22:31:50.382117 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 381: this message will repeat every 5th retry.
W20260810 22:31:51.967504 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 386: this message will repeat every 5th retry.
W20260810 22:31:52.216216 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 391: this message will repeat every 5th retry.
W20260810 22:31:52.844022 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 386: this message will repeat every 5th retry.
W20260810 22:31:53.874877 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 29 similar messages]
W20260810 22:31:54.492321 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 391: this message will repeat every 5th retry.
W20260810 22:31:54.900678 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 396: this message will repeat every 5th retry.
W20260810 22:31:55.127156 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 391: this message will repeat every 5th retry.
W20260810 22:31:57.039968 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 396: this message will repeat every 5th retry.
W20260810 22:31:57.477664 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 401: this message will repeat every 5th retry.
W20260810 22:31:57.725183 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 396: this message will repeat every 5th retry.
W20260810 22:31:59.008934 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 30 similar messages]
W20260810 22:31:59.412180 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 401: this message will repeat every 5th retry.
W20260810 22:31:59.730904 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 406: this message will repeat every 5th retry.
W20260810 22:32:00.340553 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 401: this message will repeat every 5th retry.
W20260810 22:32:02.180330 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 406: this message will repeat every 5th retry.
W20260810 22:32:02.298992 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 411: this message will repeat every 5th retry.
W20260810 22:32:03.024434 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 406: this message will repeat every 5th retry.
W20260810 22:32:04.014087 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 29 similar messages]
W20260810 22:32:04.411417 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 411: this message will repeat every 5th retry.
W20260810 22:32:04.916133 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 416: this message will repeat every 5th retry.
W20260810 22:32:05.342084 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 411: this message will repeat every 5th retry.
W20260810 22:32:07.002323 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 416: this message will repeat every 5th retry.
W20260810 22:32:07.332504 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 421: this message will repeat every 5th retry.
W20260810 22:32:07.681468 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 416: this message will repeat every 5th retry.
W20260810 22:32:09.064368 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 29 similar messages]
W20260810 22:32:09.457875 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 421: this message will repeat every 5th retry.
W20260810 22:32:09.765411 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 426: this message will repeat every 5th retry.
W20260810 22:32:10.260003 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 421: this message will repeat every 5th retry.
W20260810 22:32:11.790081 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 426: this message will repeat every 5th retry.
W20260810 22:32:12.321324 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 431: this message will repeat every 5th retry.
W20260810 22:32:12.731776 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 426: this message will repeat every 5th retry.
W20260810 22:32:14.264680 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 30 similar messages]
W20260810 22:32:14.631127 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 431: this message will repeat every 5th retry.
W20260810 22:32:14.898275 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 436: this message will repeat every 5th retry.
W20260810 22:32:15.071341 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 431: this message will repeat every 5th retry.
W20260810 22:32:17.044302 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 436: this message will repeat every 5th retry.
W20260810 22:32:17.546252 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 436: this message will repeat every 5th retry.
W20260810 22:32:17.644778 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 441: this message will repeat every 5th retry.
W20260810 22:32:19.488276 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 30 similar messages]
W20260810 22:32:19.554560 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 441: this message will repeat every 5th retry.
W20260810 22:32:20.014182 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 441: this message will repeat every 5th retry.
W20260810 22:32:20.431085 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 446: this message will repeat every 5th retry.
W20260810 22:32:22.135141 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 446: this message will repeat every 5th retry.
W20260810 22:32:22.373171 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 446: this message will repeat every 5th retry.
W20260810 22:32:22.799564 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 451: this message will repeat every 5th retry.
W20260810 22:32:24.753268 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 30 similar messages]
W20260810 22:32:24.810920 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 451: this message will repeat every 5th retry.
W20260810 22:32:24.847995 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 451: this message will repeat every 5th retry.
W20260810 22:32:25.287477 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 456: this message will repeat every 5th retry.
W20260810 22:32:27.126849 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 456: this message will repeat every 5th retry.
W20260810 22:32:27.162735 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 456: this message will repeat every 5th retry.
W20260810 22:32:27.877144 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 461: this message will repeat every 5th retry.
W20260810 22:32:29.433108 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 461: this message will repeat every 5th retry.
W20260810 22:32:29.636325 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 461: this message will repeat every 5th retry.
W20260810 22:32:29.856518 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 32 similar messages]
W20260810 22:32:30.137624 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 466: this message will repeat every 5th retry.
W20260810 22:32:31.848465 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 466: this message will repeat every 5th retry.
W20260810 22:32:32.040724 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 466: this message will repeat every 5th retry.
W20260810 22:32:32.644179 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 471: this message will repeat every 5th retry.
W20260810 22:32:34.197088 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 471: this message will repeat every 5th retry.
W20260810 22:32:34.287758 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 471: this message will repeat every 5th retry.
W20260810 22:32:35.098551 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 31 similar messages]
W20260810 22:32:35.278955 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 476: this message will repeat every 5th retry.
W20260810 22:32:35.628525 8756 heartbeater.cc:644] Failed to heartbeat to 127.8.18.190:40323 (120 consecutive failures): Network error: Failed to ping master at 127.8.18.190:40323: Client connection negotiation failed: client connection to 127.8.18.190:40323: connect: Connection refused (error 111)
W20260810 22:32:35.628995 8579 heartbeater.cc:644] Failed to heartbeat to 127.8.18.190:40323 (120 consecutive failures): Network error: Failed to ping master at 127.8.18.190:40323: Client connection negotiation failed: client connection to 127.8.18.190:40323: connect: Connection refused (error 111)
W20260810 22:32:36.689852 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 476: this message will repeat every 5th retry.
W20260810 22:32:36.860327 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 476: this message will repeat every 5th retry.
Master Summary
UUID | Address | Status
--------------------------------+--------------------+-------------
<unknown> (127.8.18.190:40323) | 127.8.18.190:40323 | UNAVAILABLE
Error from 127.8.18.190:40323: Network error: Client connection negotiation failed: client connection to 127.8.18.190:40323: connect: Connection refused (error 111) (UNAVAILABLE)
All reported replicas are:
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+----------+--------------+--------------+------------
Tablet Server Summary
Version Summary
Version | Servers
---------+---------
The cluster doesn't have any matching tablets
The cluster doesn't have any matching system tables
The cluster doesn't have any matching tables
==================
Warnings:
==================
master unusual flags check error: 1 of 1 masters were not available to retrieve unusual flags
master diverged flags check error: 1 of 1 masters were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from masters: failed to gather info from all masters: 1 of 1 had errors
Not found: master consensus error: no master consensus state available
Timed out: leader master liveness check error: Could not connect to the cluster: ConnectToClusterRpc(addrs: 127.8.18.190:40323, num_attempts: 344) passed its deadline: Network error: Client connection negotiation failed: client connection to 127.8.18.190:40323: connect: Connection refused (error 111)
FAILED
I20260810 22:32:37.058897 8266 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Timed out: leader master liveness check error: Could not connect to the cluster: ConnectToClusterRpc(addrs: 127.8.18.190:40323, num_attempts: 344) passed its deadline: Network error: Client connection negotiation failed: client connection to 127.8.18.190:40323: connect: Connection refused (error 111)
W20260810 22:32:37.735070 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 481: this message will repeat every 5th retry.
W20260810 22:32:38.986634 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 481: this message will repeat every 5th retry.
W20260810 22:32:39.320581 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 481: this message will repeat every 5th retry.
W20260810 22:32:40.160315 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 30 similar messages]
W20260810 22:32:40.405421 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 486: this message will repeat every 5th retry.
W20260810 22:32:41.528095 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 486: this message will repeat every 5th retry.
W20260810 22:32:41.636572 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 486: this message will repeat every 5th retry.
W20260810 22:32:42.763057 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 491: this message will repeat every 5th retry.
W20260810 22:32:43.924850 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 491: this message will repeat every 5th retry.
W20260810 22:32:44.191499 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 491: this message will repeat every 5th retry.
W20260810 22:32:45.172254 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 29 similar messages]
W20260810 22:32:45.176870 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 496: this message will repeat every 5th retry.
W20260810 22:32:46.200330 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 496: this message will repeat every 5th retry.
W20260810 22:32:46.611989 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 496: this message will repeat every 5th retry.
W20260810 22:32:47.341617 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 501: this message will repeat every 5th retry.
W20260810 22:32:48.472074 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 501: this message will repeat every 5th retry.
W20260810 22:32:48.938160 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 501: this message will repeat every 5th retry.
W20260810 22:32:49.931316 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 506: this message will repeat every 5th retry.
W20260810 22:32:50.329388 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 32 similar messages]
W20260810 22:32:51.124794 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 506: this message will repeat every 5th retry.
W20260810 22:32:51.410594 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 506: this message will repeat every 5th retry.
W20260810 22:32:52.278185 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 511: this message will repeat every 5th retry.
W20260810 22:32:53.696282 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 511: this message will repeat every 5th retry.
W20260810 22:32:54.051590 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 511: this message will repeat every 5th retry.
W20260810 22:32:54.975009 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 516: this message will repeat every 5th retry.
W20260810 22:32:55.380563 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 29 similar messages]
W20260810 22:32:56.077824 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 516: this message will repeat every 5th retry.
W20260810 22:32:56.331758 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 516: this message will repeat every 5th retry.
W20260810 22:32:57.257576 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 521: this message will repeat every 5th retry.
W20260810 22:32:58.535848 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 521: this message will repeat every 5th retry.
W20260810 22:32:58.991349 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 521: this message will repeat every 5th retry.
W20260810 22:32:59.834205 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 526: this message will repeat every 5th retry.
W20260810 22:33:00.559203 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 30 similar messages]
W20260810 22:33:01.021924 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 526: this message will repeat every 5th retry.
W20260810 22:33:01.499341 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 526: this message will repeat every 5th retry.
W20260810 22:33:02.142189 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 531: this message will repeat every 5th retry.
W20260810 22:33:03.632320 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 531: this message will repeat every 5th retry.
W20260810 22:33:03.836362 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 531: this message will repeat every 5th retry.
W20260810 22:33:04.676748 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 536: this message will repeat every 5th retry.
W20260810 22:33:05.600904 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 31 similar messages]
W20260810 22:33:05.976907 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 536: this message will repeat every 5th retry.
W20260810 22:33:06.409855 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 536: this message will repeat every 5th retry.
W20260810 22:33:07.270082 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 541: this message will repeat every 5th retry.
W20260810 22:33:08.343896 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 541: this message will repeat every 5th retry.
W20260810 22:33:08.903991 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 541: this message will repeat every 5th retry.
W20260810 22:33:09.896350 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 546: this message will repeat every 5th retry.
W20260810 22:33:10.840027 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 29 similar messages]
W20260810 22:33:10.853597 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 546: this message will repeat every 5th retry.
W20260810 22:33:11.436503 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 546: this message will repeat every 5th retry.
W20260810 22:33:12.283062 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 551: this message will repeat every 5th retry.
W20260810 22:33:13.679843 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 551: this message will repeat every 5th retry.
W20260810 22:33:13.680299 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 551: this message will repeat every 5th retry.
W20260810 22:33:14.756212 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 556: this message will repeat every 5th retry.
W20260810 22:33:15.889892 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 30 similar messages]
W20260810 22:33:16.165735 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 556: this message will repeat every 5th retry.
W20260810 22:33:16.429777 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 556: this message will repeat every 5th retry.
W20260810 22:33:17.098084 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 561: this message will repeat every 5th retry.
W20260810 22:33:18.609284 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 561: this message will repeat every 5th retry.
W20260810 22:33:19.005460 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 561: this message will repeat every 5th retry.
W20260810 22:33:19.288991 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 566: this message will repeat every 5th retry.
W20260810 22:33:20.792873 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 566: this message will repeat every 5th retry.
W20260810 22:33:21.093151 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 31 similar messages]
W20260810 22:33:21.676026 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 566: this message will repeat every 5th retry.
W20260810 22:33:21.943073 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 571: this message will repeat every 5th retry.
W20260810 22:33:23.172724 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 571: this message will repeat every 5th retry.
W20260810 22:33:24.279126 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 571: this message will repeat every 5th retry.
W20260810 22:33:24.448680 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 576: this message will repeat every 5th retry.
W20260810 22:33:25.523867 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 576: this message will repeat every 5th retry.
W20260810 22:33:26.248602 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 30 similar messages]
W20260810 22:33:26.773929 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 576: this message will repeat every 5th retry.
W20260810 22:33:27.132895 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 581: this message will repeat every 5th retry.
W20260810 22:33:28.033298 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 581: this message will repeat every 5th retry.
W20260810 22:33:29.297950 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 581: this message will repeat every 5th retry.
W20260810 22:33:29.948442 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 586: this message will repeat every 5th retry.
W20260810 22:33:30.675602 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 586: this message will repeat every 5th retry.
W20260810 22:33:31.347734 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 28 similar messages]
W20260810 22:33:31.933696 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 586: this message will repeat every 5th retry.
W20260810 22:33:32.465771 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 591: this message will repeat every 5th retry.
W20260810 22:33:33.179949 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 591: this message will repeat every 5th retry.
W20260810 22:33:34.378860 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 591: this message will repeat every 5th retry.
W20260810 22:33:34.887838 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 596: this message will repeat every 5th retry.
W20260810 22:33:35.532322 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 596: this message will repeat every 5th retry.
W20260810 22:33:35.700012 8756 heartbeater.cc:644] Failed to heartbeat to 127.8.18.190:40323 (180 consecutive failures): Network error: Failed to ping master at 127.8.18.190:40323: Client connection negotiation failed: client connection to 127.8.18.190:40323: connect: Connection refused (error 111)
W20260810 22:33:35.700764 8579 heartbeater.cc:644] Failed to heartbeat to 127.8.18.190:40323 (180 consecutive failures): Network error: Failed to ping master at 127.8.18.190:40323: Client connection negotiation failed: client connection to 127.8.18.190:40323: connect: Connection refused (error 111)
W20260810 22:33:36.481304 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 30 similar messages]
W20260810 22:33:36.896934 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 596: this message will repeat every 5th retry.
W20260810 22:33:37.627888 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 601: this message will repeat every 5th retry.
W20260810 22:33:37.742790 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 601: this message will repeat every 5th retry.
I20260810 22:33:38.197050 13498 consensus_queue.cc:579] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 [LEADER]: Leader has been unable to successfully communicate with peer d06d8f442c824758ae944ffe71e3cb44 for more than 300 seconds (300.325s)
Master Summary
UUID | Address | Status
--------------------------------+--------------------+-------------
<unknown> (127.8.18.190:40323) | 127.8.18.190:40323 | UNAVAILABLE
Error from 127.8.18.190:40323: Network error: Client connection negotiation failed: client connection to 127.8.18.190:40323: connect: Connection refused (error 111) (UNAVAILABLE)
All reported replicas are:
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+----------+--------------+--------------+------------
Tablet Server Summary
Version Summary
Version | Servers
---------+---------
The cluster doesn't have any matching tablets
The cluster doesn't have any matching system tables
The cluster doesn't have any matching tables
==================
Warnings:
==================
master unusual flags check error: 1 of 1 masters were not available to retrieve unusual flags
master diverged flags check error: 1 of 1 masters were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from masters: failed to gather info from all masters: 1 of 1 had errors
Not found: master consensus error: no master consensus state available
Timed out: leader master liveness check error: Could not connect to the cluster: ConnectToClusterRpc(addrs: 127.8.18.190:40323, num_attempts: 344) passed its deadline: Network error: Client connection negotiation failed: client connection to 127.8.18.190:40323: connect: Connection refused (error 111)
FAILED
I20260810 22:33:38.253583 8266 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Timed out: leader master liveness check error: Could not connect to the cluster: ConnectToClusterRpc(addrs: 127.8.18.190:40323, num_attempts: 344) passed its deadline: Network error: Client connection negotiation failed: client connection to 127.8.18.190:40323: connect: Connection refused (error 111)
I20260810 22:33:38.347489 13467 consensus_queue.cc:579] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 [LEADER]: Leader has been unable to successfully communicate with peer d06d8f442c824758ae944ffe71e3cb44 for more than 300 seconds (300.357s)
W20260810 22:33:39.380110 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 601: this message will repeat every 5th retry.
I20260810 22:33:39.939684 13467 consensus_queue.cc:579] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 [LEADER]: Leader has been unable to successfully communicate with peer d06d8f442c824758ae944ffe71e3cb44 for more than 300 seconds (300.222s)
W20260810 22:33:40.180301 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 606: this message will repeat every 5th retry.
W20260810 22:33:40.295542 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 606: this message will repeat every 5th retry.
W20260810 22:33:41.635380 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 30 similar messages]
W20260810 22:33:41.997424 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 606: this message will repeat every 5th retry.
W20260810 22:33:42.662739 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 611: this message will repeat every 5th retry.
W20260810 22:33:42.796310 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 611: this message will repeat every 5th retry.
W20260810 22:33:44.524456 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 611: this message will repeat every 5th retry.
W20260810 22:33:45.343938 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 616: this message will repeat every 5th retry.
W20260810 22:33:45.416189 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 616: this message will repeat every 5th retry.
W20260810 22:33:46.855656 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 29 similar messages]
W20260810 22:33:46.985174 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 616: this message will repeat every 5th retry.
W20260810 22:33:47.788566 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 621: this message will repeat every 5th retry.
W20260810 22:33:47.937757 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 621: this message will repeat every 5th retry.
W20260810 22:33:49.545589 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 621: this message will repeat every 5th retry.
W20260810 22:33:49.950145 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 626: this message will repeat every 5th retry.
W20260810 22:33:50.777830 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 626: this message will repeat every 5th retry.
W20260810 22:33:51.831517 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 626: this message will repeat every 5th retry.
W20260810 22:33:52.146258 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 31 similar messages]
W20260810 22:33:52.744719 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 631: this message will repeat every 5th retry.
W20260810 22:33:53.129449 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 631: this message will repeat every 5th retry.
W20260810 22:33:54.276182 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 631: this message will repeat every 5th retry.
W20260810 22:33:55.082290 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 636: this message will repeat every 5th retry.
W20260810 22:33:55.656270 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 636: this message will repeat every 5th retry.
W20260810 22:33:56.651625 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 636: this message will repeat every 5th retry.
W20260810 22:33:57.217300 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 30 similar messages]
W20260810 22:33:57.699025 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 641: this message will repeat every 5th retry.
W20260810 22:33:57.919080 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 641: this message will repeat every 5th retry.
W20260810 22:33:59.386267 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 641: this message will repeat every 5th retry.
W20260810 22:34:00.013999 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 646: this message will repeat every 5th retry.
W20260810 22:34:00.293159 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 646: this message will repeat every 5th retry.
W20260810 22:34:01.638581 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 646: this message will repeat every 5th retry.
W20260810 22:34:02.221791 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 30 similar messages]
W20260810 22:34:02.426219 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 651: this message will repeat every 5th retry.
W20260810 22:34:02.950802 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 651: this message will repeat every 5th retry.
W20260810 22:34:04.285687 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 651: this message will repeat every 5th retry.
W20260810 22:34:05.138103 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 656: this message will repeat every 5th retry.
W20260810 22:34:05.515528 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 656: this message will repeat every 5th retry.
W20260810 22:34:06.753849 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 656: this message will repeat every 5th retry.
W20260810 22:34:07.528038 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 30 similar messages]
W20260810 22:34:07.528954 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 661: this message will repeat every 5th retry.
W20260810 22:34:07.908723 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 661: this message will repeat every 5th retry.
W20260810 22:34:08.966533 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 661: this message will repeat every 5th retry.
W20260810 22:34:09.855161 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 666: this message will repeat every 5th retry.
W20260810 22:34:10.472261 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 666: this message will repeat every 5th retry.
W20260810 22:34:11.211647 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 666: this message will repeat every 5th retry.
W20260810 22:34:12.558588 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 31 similar messages]
W20260810 22:34:12.559260 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 671: this message will repeat every 5th retry.
W20260810 22:34:12.743014 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 671: this message will repeat every 5th retry.
W20260810 22:34:13.700378 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 671: this message will repeat every 5th retry.
W20260810 22:34:15.176997 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 676: this message will repeat every 5th retry.
W20260810 22:34:15.223233 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 676: this message will repeat every 5th retry.
W20260810 22:34:15.909260 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 676: this message will repeat every 5th retry.
W20260810 22:34:17.842386 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 30 similar messages]
W20260810 22:34:17.843190 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 681: this message will repeat every 5th retry.
W20260810 22:34:17.933741 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 681: this message will repeat every 5th retry.
W20260810 22:34:18.465423 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 681: this message will repeat every 5th retry.
W20260810 22:34:20.338490 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 686: this message will repeat every 5th retry.
W20260810 22:34:20.556747 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 686: this message will repeat every 5th retry.
W20260810 22:34:21.047092 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 686: this message will repeat every 5th retry.
W20260810 22:34:22.807125 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 691: this message will repeat every 5th retry.
W20260810 22:34:23.069504 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 30 similar messages]
W20260810 22:34:23.070307 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 691: this message will repeat every 5th retry.
W20260810 22:34:23.676036 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 691: this message will repeat every 5th retry.
W20260810 22:34:25.244451 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 696: this message will repeat every 5th retry.
W20260810 22:34:25.538784 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 696: this message will repeat every 5th retry.
W20260810 22:34:26.244643 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 696: this message will repeat every 5th retry.
W20260810 22:34:27.865636 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 701: this message will repeat every 5th retry.
W20260810 22:34:28.231803 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 29 similar messages]
W20260810 22:34:28.232601 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 701: this message will repeat every 5th retry.
W20260810 22:34:28.692389 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 701: this message will repeat every 5th retry.
W20260810 22:34:30.267167 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 706: this message will repeat every 5th retry.
W20260810 22:34:30.355613 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 706: this message will repeat every 5th retry.
W20260810 22:34:31.080384 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 706: this message will repeat every 5th retry.
W20260810 22:34:32.572228 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 711: this message will repeat every 5th retry.
W20260810 22:34:32.915930 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 711: this message will repeat every 5th retry.
W20260810 22:34:33.425622 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 32 similar messages]
W20260810 22:34:33.426386 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 711: this message will repeat every 5th retry.
W20260810 22:34:35.076376 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 716: this message will repeat every 5th retry.
W20260810 22:34:35.336467 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 716: this message will repeat every 5th retry.
W20260810 22:34:35.672322 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 716: this message will repeat every 5th retry.
W20260810 22:34:35.778653 8579 heartbeater.cc:644] Failed to heartbeat to 127.8.18.190:40323 (240 consecutive failures): Network error: Failed to ping master at 127.8.18.190:40323: Client connection negotiation failed: client connection to 127.8.18.190:40323: connect: Connection refused (error 111)
W20260810 22:34:36.003531 8756 heartbeater.cc:644] Failed to heartbeat to 127.8.18.190:40323 (242 consecutive failures): Network error: Failed to ping master at 127.8.18.190:40323: Client connection negotiation failed: client connection to 127.8.18.190:40323: connect: Connection refused (error 111)
W20260810 22:34:37.493150 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 721: this message will repeat every 5th retry.
W20260810 22:34:37.968345 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 721: this message will repeat every 5th retry.
W20260810 22:34:38.248821 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 721: this message will repeat every 5th retry.
W20260810 22:34:38.502763 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 31 similar messages]
Master Summary
UUID | Address | Status
--------------------------------+--------------------+-------------
<unknown> (127.8.18.190:40323) | 127.8.18.190:40323 | UNAVAILABLE
Error from 127.8.18.190:40323: Network error: Client connection negotiation failed: client connection to 127.8.18.190:40323: connect: Connection refused (error 111) (UNAVAILABLE)
All reported replicas are:
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+----------+--------------+--------------+------------
Tablet Server Summary
Version Summary
Version | Servers
---------+---------
The cluster doesn't have any matching tablets
The cluster doesn't have any matching system tables
The cluster doesn't have any matching tables
==================
Warnings:
==================
master unusual flags check error: 1 of 1 masters were not available to retrieve unusual flags
master diverged flags check error: 1 of 1 masters were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from masters: failed to gather info from all masters: 1 of 1 had errors
Not found: master consensus error: no master consensus state available
Timed out: leader master liveness check error: Could not connect to the cluster: ConnectToClusterRpc(addrs: 127.8.18.190:40323, num_attempts: 344) passed its deadline: Network error: Client connection negotiation failed: client connection to 127.8.18.190:40323: connect: Connection refused (error 111)
FAILED
I20260810 22:34:39.478559 8266 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Timed out: leader master liveness check error: Could not connect to the cluster: ConnectToClusterRpc(addrs: 127.8.18.190:40323, num_attempts: 344) passed its deadline: Network error: Client connection negotiation failed: client connection to 127.8.18.190:40323: connect: Connection refused (error 111)
W20260810 22:34:39.852399 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 726: this message will repeat every 5th retry.
W20260810 22:34:40.553849 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 726: this message will repeat every 5th retry.
W20260810 22:34:40.894357 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 726: this message will repeat every 5th retry.
W20260810 22:34:42.182448 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 731: this message will repeat every 5th retry.
W20260810 22:34:43.229766 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 731: this message will repeat every 5th retry.
W20260810 22:34:43.293196 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 731: this message will repeat every 5th retry.
W20260810 22:34:43.669870 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 29 similar messages]
W20260810 22:34:44.707945 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 736: this message will repeat every 5th retry.
W20260810 22:34:45.636260 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 736: this message will repeat every 5th retry.
W20260810 22:34:45.763628 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 736: this message will repeat every 5th retry.
W20260810 22:34:47.519630 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 741: this message will repeat every 5th retry.
W20260810 22:34:48.199903 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 741: this message will repeat every 5th retry.
W20260810 22:34:48.224901 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 741: this message will repeat every 5th retry.
W20260810 22:34:48.679169 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 29 similar messages]
W20260810 22:34:50.242230 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 746: this message will repeat every 5th retry.
W20260810 22:34:50.468904 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 746: this message will repeat every 5th retry.
W20260810 22:34:50.651947 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 746: this message will repeat every 5th retry.
W20260810 22:34:52.590111 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 751: this message will repeat every 5th retry.
W20260810 22:34:53.058166 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 751: this message will repeat every 5th retry.
W20260810 22:34:53.125559 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 751: this message will repeat every 5th retry.
W20260810 22:34:54.061410 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 31 similar messages]
W20260810 22:34:55.060949 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 756: this message will repeat every 5th retry.
W20260810 22:34:55.363771 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 756: this message will repeat every 5th retry.
W20260810 22:34:55.520709 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 756: this message will repeat every 5th retry.
W20260810 22:34:57.544414 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 761: this message will repeat every 5th retry.
W20260810 22:34:57.753734 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 761: this message will repeat every 5th retry.
W20260810 22:34:58.110901 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 761: this message will repeat every 5th retry.
W20260810 22:34:59.107311 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 32 similar messages]
W20260810 22:34:59.732336 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 766: this message will repeat every 5th retry.
W20260810 22:34:59.895133 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 766: this message will repeat every 5th retry.
W20260810 22:35:00.701280 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 766: this message will repeat every 5th retry.
W20260810 22:35:01.987134 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 771: this message will repeat every 5th retry.
W20260810 22:35:02.198807 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 771: this message will repeat every 5th retry.
W20260810 22:35:03.440212 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 771: this message will repeat every 5th retry.
W20260810 22:35:04.279839 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 30 similar messages]
W20260810 22:35:04.280563 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 776: this message will repeat every 5th retry.
W20260810 22:35:04.720844 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 776: this message will repeat every 5th retry.
W20260810 22:35:06.096060 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 776: this message will repeat every 5th retry.
W20260810 22:35:06.793649 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 781: this message will repeat every 5th retry.
W20260810 22:35:06.924185 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 781: this message will repeat every 5th retry.
W20260810 22:35:08.692061 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 781: this message will repeat every 5th retry.
W20260810 22:35:09.204636 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 786: this message will repeat every 5th retry.
W20260810 22:35:09.413911 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 31 similar messages]
W20260810 22:35:09.414774 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 786: this message will repeat every 5th retry.
W20260810 22:35:11.330376 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 786: this message will repeat every 5th retry.
W20260810 22:35:11.751250 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 791: this message will repeat every 5th retry.
W20260810 22:35:11.867573 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 791: this message will repeat every 5th retry.
W20260810 22:35:13.742146 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 791: this message will repeat every 5th retry.
W20260810 22:35:13.954962 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 796: this message will repeat every 5th retry.
W20260810 22:35:14.351717 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 796: this message will repeat every 5th retry.
W20260810 22:35:14.760344 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 31 similar messages]
W20260810 22:35:16.352411 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 796: this message will repeat every 5th retry.
W20260810 22:35:16.352936 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 801: this message will repeat every 5th retry.
W20260810 22:35:16.572227 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 801: this message will repeat every 5th retry.
W20260810 22:35:18.991021 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 806: this message will repeat every 5th retry.
W20260810 22:35:19.020869 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 801: this message will repeat every 5th retry.
W20260810 22:35:19.150696 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 806: this message will repeat every 5th retry.
W20260810 22:35:19.923786 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 30 similar messages]
W20260810 22:35:21.378623 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 811: this message will repeat every 5th retry.
W20260810 22:35:21.535311 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 806: this message will repeat every 5th retry.
W20260810 22:35:21.635598 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 811: this message will repeat every 5th retry.
W20260810 22:35:23.675244 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 816: this message will repeat every 5th retry.
W20260810 22:35:24.181707 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 816: this message will repeat every 5th retry.
W20260810 22:35:24.237747 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 811: this message will repeat every 5th retry.
W20260810 22:35:24.945448 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 30 similar messages]
W20260810 22:35:26.157992 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 821: this message will repeat every 5th retry.
W20260810 22:35:26.602620 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 821: this message will repeat every 5th retry.
W20260810 22:35:26.619854 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 816: this message will repeat every 5th retry.
W20260810 22:35:28.761642 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 826: this message will repeat every 5th retry.
W20260810 22:35:29.274724 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 826: this message will repeat every 5th retry.
W20260810 22:35:29.274824 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 821: this message will repeat every 5th retry.
W20260810 22:35:30.171350 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 29 similar messages]
W20260810 22:35:31.260645 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 831: this message will repeat every 5th retry.
W20260810 22:35:31.678594 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 831: this message will repeat every 5th retry.
W20260810 22:35:31.893374 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 826: this message will repeat every 5th retry.
W20260810 22:35:33.637730 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 836: this message will repeat every 5th retry.
W20260810 22:35:34.184126 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 836: this message will repeat every 5th retry.
W20260810 22:35:34.398341 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 831: this message will repeat every 5th retry.
W20260810 22:35:35.184430 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 30 similar messages]
W20260810 22:35:35.857229 8579 heartbeater.cc:644] Failed to heartbeat to 127.8.18.190:40323 (300 consecutive failures): Network error: Failed to ping master at 127.8.18.190:40323: Client connection negotiation failed: client connection to 127.8.18.190:40323: connect: Connection refused (error 111)
W20260810 22:35:35.941642 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 841: this message will repeat every 5th retry.
W20260810 22:35:36.074317 8756 heartbeater.cc:644] Failed to heartbeat to 127.8.18.190:40323 (302 consecutive failures): Network error: Failed to ping master at 127.8.18.190:40323: Client connection negotiation failed: client connection to 127.8.18.190:40323: connect: Connection refused (error 111)
W20260810 22:35:36.576895 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 841: this message will repeat every 5th retry.
W20260810 22:35:36.814528 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 836: this message will repeat every 5th retry.
W20260810 22:35:38.453104 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 846: this message will repeat every 5th retry.
W20260810 22:35:38.937071 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 846: this message will repeat every 5th retry.
W20260810 22:35:39.370219 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 841: this message will repeat every 5th retry.
W20260810 22:35:40.279673 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 30 similar messages]
Master Summary
UUID | Address | Status
--------------------------------+--------------------+-------------
<unknown> (127.8.18.190:40323) | 127.8.18.190:40323 | UNAVAILABLE
Error from 127.8.18.190:40323: Network error: Client connection negotiation failed: client connection to 127.8.18.190:40323: connect: Connection refused (error 111) (UNAVAILABLE)
All reported replicas are:
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+----------+--------------+--------------+------------
Tablet Server Summary
Version Summary
Version | Servers
---------+---------
The cluster doesn't have any matching tablets
The cluster doesn't have any matching system tables
The cluster doesn't have any matching tables
==================
Warnings:
==================
master unusual flags check error: 1 of 1 masters were not available to retrieve unusual flags
master diverged flags check error: 1 of 1 masters were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from masters: failed to gather info from all masters: 1 of 1 had errors
Not found: master consensus error: no master consensus state available
Timed out: leader master liveness check error: Could not connect to the cluster: ConnectToClusterRpc(addrs: 127.8.18.190:40323, num_attempts: 344) passed its deadline: Network error: Client connection negotiation failed: client connection to 127.8.18.190:40323: connect: Connection refused (error 111)
FAILED
I20260810 22:35:40.705392 8266 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Timed out: leader master liveness check error: Could not connect to the cluster: ConnectToClusterRpc(addrs: 127.8.18.190:40323, num_attempts: 344) passed its deadline: Network error: Client connection negotiation failed: client connection to 127.8.18.190:40323: connect: Connection refused (error 111)
W20260810 22:35:40.754065 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 851: this message will repeat every 5th retry.
W20260810 22:35:41.266741 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 851: this message will repeat every 5th retry.
W20260810 22:35:41.585527 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 846: this message will repeat every 5th retry.
W20260810 22:35:43.102950 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 856: this message will repeat every 5th retry.
W20260810 22:35:43.710147 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 856: this message will repeat every 5th retry.
W20260810 22:35:44.085078 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 851: this message will repeat every 5th retry.
W20260810 22:35:45.301621 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 30 similar messages]
W20260810 22:35:45.786798 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 861: this message will repeat every 5th retry.
W20260810 22:35:46.497164 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 861: this message will repeat every 5th retry.
W20260810 22:35:46.528934 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 856: this message will repeat every 5th retry.
W20260810 22:35:47.996762 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 866: this message will repeat every 5th retry.
W20260810 22:35:48.742916 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 866: this message will repeat every 5th retry.
W20260810 22:35:48.884162 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 861: this message will repeat every 5th retry.
W20260810 22:35:50.307417 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 30 similar messages]
W20260810 22:35:50.537978 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 871: this message will repeat every 5th retry.
W20260810 22:35:51.358273 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 866: this message will repeat every 5th retry.
W20260810 22:35:51.395445 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 871: this message will repeat every 5th retry.
W20260810 22:35:53.186928 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 876: this message will repeat every 5th retry.
W20260810 22:35:54.087276 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 871: this message will repeat every 5th retry.
W20260810 22:35:54.102416 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 876: this message will repeat every 5th retry.
W20260810 22:35:55.364805 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 28 similar messages]
W20260810 22:35:55.933142 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 881: this message will repeat every 5th retry.
W20260810 22:35:56.615444 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 876: this message will repeat every 5th retry.
W20260810 22:35:56.622377 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 881: this message will repeat every 5th retry.
W20260810 22:35:58.466939 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 886: this message will repeat every 5th retry.
W20260810 22:35:59.228442 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 881: this message will repeat every 5th retry.
W20260810 22:35:59.332247 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 886: this message will repeat every 5th retry.
W20260810 22:36:00.430372 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 28 similar messages]
W20260810 22:36:01.160665 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 891: this message will repeat every 5th retry.
W20260810 22:36:01.616652 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 886: this message will repeat every 5th retry.
W20260810 22:36:02.033464 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 891: this message will repeat every 5th retry.
W20260810 22:36:03.625618 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 896: this message will repeat every 5th retry.
W20260810 22:36:03.976675 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 891: this message will repeat every 5th retry.
W20260810 22:36:04.645699 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 896: this message will repeat every 5th retry.
W20260810 22:36:05.581670 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 30 similar messages]
W20260810 22:36:06.011049 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 901: this message will repeat every 5th retry.
W20260810 22:36:06.342146 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 896: this message will repeat every 5th retry.
W20260810 22:36:06.967612 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 901: this message will repeat every 5th retry.
W20260810 22:36:08.766669 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 906: this message will repeat every 5th retry.
W20260810 22:36:08.860930 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 901: this message will repeat every 5th retry.
W20260810 22:36:09.508586 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 906: this message will repeat every 5th retry.
W20260810 22:36:10.626585 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 29 similar messages]
W20260810 22:36:11.142020 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 906: this message will repeat every 5th retry.
W20260810 22:36:11.173323 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 911: this message will repeat every 5th retry.
W20260810 22:36:12.040140 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 911: this message will repeat every 5th retry.
W20260810 22:36:13.436164 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 916: this message will repeat every 5th retry.
W20260810 22:36:13.551940 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 911: this message will repeat every 5th retry.
W20260810 22:36:14.306931 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 916: this message will repeat every 5th retry.
W20260810 22:36:15.646862 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 31 similar messages]
W20260810 22:36:15.989058 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 921: this message will repeat every 5th retry.
W20260810 22:36:16.075022 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 916: this message will repeat every 5th retry.
W20260810 22:36:16.918304 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 921: this message will repeat every 5th retry.
W20260810 22:36:18.565251 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 926: this message will repeat every 5th retry.
W20260810 22:36:18.790061 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 921: this message will repeat every 5th retry.
W20260810 22:36:19.407382 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 926: this message will repeat every 5th retry.
W20260810 22:36:20.810132 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 30 similar messages]
W20260810 22:36:20.810894 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 926: this message will repeat every 5th retry.
W20260810 22:36:21.227994 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 931: this message will repeat every 5th retry.
W20260810 22:36:21.948145 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 931: this message will repeat every 5th retry.
W20260810 22:36:23.238008 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 931: this message will repeat every 5th retry.
W20260810 22:36:23.822082 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 936: this message will repeat every 5th retry.
W20260810 22:36:24.381013 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 936: this message will repeat every 5th retry.
W20260810 22:36:25.771293 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 936: this message will repeat every 5th retry.
W20260810 22:36:26.125054 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 31 similar messages]
W20260810 22:36:26.125845 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 941: this message will repeat every 5th retry.
W20260810 22:36:26.742846 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 941: this message will repeat every 5th retry.
W20260810 22:36:28.137529 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 941: this message will repeat every 5th retry.
W20260810 22:36:28.703186 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 946: this message will repeat every 5th retry.
W20260810 22:36:29.260902 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 946: this message will repeat every 5th retry.
W20260810 22:36:30.619311 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 946: this message will repeat every 5th retry.
W20260810 22:36:31.031920 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 951: this message will repeat every 5th retry.
W20260810 22:36:31.396090 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 31 similar messages]
W20260810 22:36:31.842674 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 951: this message will repeat every 5th retry.
W20260810 22:36:33.149871 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 951: this message will repeat every 5th retry.
W20260810 22:36:33.737519 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 956: this message will repeat every 5th retry.
W20260810 22:36:34.552050 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 956: this message will repeat every 5th retry.
W20260810 22:36:35.689241 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 956: this message will repeat every 5th retry.
W20260810 22:36:35.935281 8579 heartbeater.cc:644] Failed to heartbeat to 127.8.18.190:40323 (360 consecutive failures): Network error: Failed to ping master at 127.8.18.190:40323: Client connection negotiation failed: client connection to 127.8.18.190:40323: connect: Connection refused (error 111)
W20260810 22:36:36.140836 8756 heartbeater.cc:644] Failed to heartbeat to 127.8.18.190:40323 (362 consecutive failures): Network error: Failed to ping master at 127.8.18.190:40323: Client connection negotiation failed: client connection to 127.8.18.190:40323: connect: Connection refused (error 111)
W20260810 22:36:36.275501 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 961: this message will repeat every 5th retry.
W20260810 22:36:36.417008 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 29 similar messages]
W20260810 22:36:36.862140 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 961: this message will repeat every 5th retry.
W20260810 22:36:38.345101 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 961: this message will repeat every 5th retry.
W20260810 22:36:38.751897 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 966: this message will repeat every 5th retry.
W20260810 22:36:39.344461 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 966: this message will repeat every 5th retry.
W20260810 22:36:40.873250 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 966: this message will repeat every 5th retry.
W20260810 22:36:41.122573 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 971: this message will repeat every 5th retry.
W20260810 22:36:41.563686 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 30 similar messages]
W20260810 22:36:41.675951 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 971: this message will repeat every 5th retry.
Master Summary
UUID | Address | Status
--------------------------------+--------------------+-------------
<unknown> (127.8.18.190:40323) | 127.8.18.190:40323 | UNAVAILABLE
Error from 127.8.18.190:40323: Network error: Client connection negotiation failed: client connection to 127.8.18.190:40323: connect: Connection refused (error 111) (UNAVAILABLE)
All reported replicas are:
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+----------+--------------+--------------+------------
Tablet Server Summary
Version Summary
Version | Servers
---------+---------
The cluster doesn't have any matching tablets
The cluster doesn't have any matching system tables
The cluster doesn't have any matching tables
==================
Warnings:
==================
master unusual flags check error: 1 of 1 masters were not available to retrieve unusual flags
master diverged flags check error: 1 of 1 masters were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from masters: failed to gather info from all masters: 1 of 1 had errors
Not found: master consensus error: no master consensus state available
Timed out: leader master liveness check error: Could not connect to the cluster: ConnectToClusterRpc(addrs: 127.8.18.190:40323, num_attempts: 344) passed its deadline: Network error: Client connection negotiation failed: client connection to 127.8.18.190:40323: connect: Connection refused (error 111)
FAILED
I20260810 22:36:41.867304 8266 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Timed out: leader master liveness check error: Could not connect to the cluster: ConnectToClusterRpc(addrs: 127.8.18.190:40323, num_attempts: 344) passed its deadline: Network error: Client connection negotiation failed: client connection to 127.8.18.190:40323: connect: Connection refused (error 111)
W20260810 22:36:43.099663 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 971: this message will repeat every 5th retry.
W20260810 22:36:43.448738 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 976: this message will repeat every 5th retry.
W20260810 22:36:44.460131 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 976: this message will repeat every 5th retry.
W20260810 22:36:45.728802 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 976: this message will repeat every 5th retry.
W20260810 22:36:45.874665 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 981: this message will repeat every 5th retry.
W20260810 22:36:46.703939 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 30 similar messages]
W20260810 22:36:46.899035 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 981: this message will repeat every 5th retry.
W20260810 22:36:48.247460 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 981: this message will repeat every 5th retry.
W20260810 22:36:48.477375 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 986: this message will repeat every 5th retry.
W20260810 22:36:49.193539 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 986: this message will repeat every 5th retry.
W20260810 22:36:50.645385 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 986: this message will repeat every 5th retry.
W20260810 22:36:50.996564 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 991: this message will repeat every 5th retry.
W20260810 22:36:51.801574 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 30 similar messages]
W20260810 22:36:51.802371 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 991: this message will repeat every 5th retry.
W20260810 22:36:53.091461 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 991: this message will repeat every 5th retry.
W20260810 22:36:53.374218 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 996: this message will repeat every 5th retry.
W20260810 22:36:54.240851 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 996: this message will repeat every 5th retry.
W20260810 22:36:55.678970 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 996: this message will repeat every 5th retry.
W20260810 22:36:56.011801 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1001: this message will repeat every 5th retry.
W20260810 22:36:56.589531 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1001: this message will repeat every 5th retry.
W20260810 22:36:57.039830 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 30 similar messages]
W20260810 22:36:58.252287 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1001: this message will repeat every 5th retry.
W20260810 22:36:58.523823 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1006: this message will repeat every 5th retry.
W20260810 22:36:59.161710 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1006: this message will repeat every 5th retry.
W20260810 22:37:00.748027 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1006: this message will repeat every 5th retry.
W20260810 22:37:01.033010 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1011: this message will repeat every 5th retry.
W20260810 22:37:01.886701 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1011: this message will repeat every 5th retry.
W20260810 22:37:02.164356 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 29 similar messages]
W20260810 22:37:03.466408 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1011: this message will repeat every 5th retry.
W20260810 22:37:03.885504 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1016: this message will repeat every 5th retry.
W20260810 22:37:04.326347 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1016: this message will repeat every 5th retry.
W20260810 22:37:06.092052 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1016: this message will repeat every 5th retry.
W20260810 22:37:06.509864 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1021: this message will repeat every 5th retry.
W20260810 22:37:06.878481 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1021: this message will repeat every 5th retry.
W20260810 22:37:07.366608 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 29 similar messages]
W20260810 22:37:08.448410 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1021: this message will repeat every 5th retry.
W20260810 22:37:08.700238 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1026: this message will repeat every 5th retry.
W20260810 22:37:09.603951 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1026: this message will repeat every 5th retry.
W20260810 22:37:11.225294 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1031: this message will repeat every 5th retry.
W20260810 22:37:11.310853 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1026: this message will repeat every 5th retry.
W20260810 22:37:12.181449 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1031: this message will repeat every 5th retry.
W20260810 22:37:12.411422 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 29 similar messages]
W20260810 22:37:13.750377 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1036: this message will repeat every 5th retry.
W20260810 22:37:14.085320 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1031: this message will repeat every 5th retry.
W20260810 22:37:14.757735 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1036: this message will repeat every 5th retry.
W20260810 22:37:16.019102 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1041: this message will repeat every 5th retry.
W20260810 22:37:16.514395 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1036: this message will repeat every 5th retry.
W20260810 22:37:17.324062 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1041: this message will repeat every 5th retry.
W20260810 22:37:17.474380 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 29 similar messages]
W20260810 22:37:18.631035 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1046: this message will repeat every 5th retry.
W20260810 22:37:18.749737 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1041: this message will repeat every 5th retry.
W20260810 22:37:19.936091 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1046: this message will repeat every 5th retry.
W20260810 22:37:21.077446 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1051: this message will repeat every 5th retry.
W20260810 22:37:21.197651 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1046: this message will repeat every 5th retry.
W20260810 22:37:22.415612 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1051: this message will repeat every 5th retry.
W20260810 22:37:22.538942 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 30 similar messages]
W20260810 22:37:23.501083 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1056: this message will repeat every 5th retry.
W20260810 22:37:23.721494 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1051: this message will repeat every 5th retry.
W20260810 22:37:25.122079 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1056: this message will repeat every 5th retry.
W20260810 22:37:26.068662 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1061: this message will repeat every 5th retry.
W20260810 22:37:26.246598 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1056: this message will repeat every 5th retry.
W20260810 22:37:27.484934 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1061: this message will repeat every 5th retry.
W20260810 22:37:27.572391 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 30 similar messages]
W20260810 22:37:28.468403 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1066: this message will repeat every 5th retry.
W20260810 22:37:28.587352 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1061: this message will repeat every 5th retry.
W20260810 22:37:29.846650 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1066: this message will repeat every 5th retry.
W20260810 22:37:30.918871 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1071: this message will repeat every 5th retry.
W20260810 22:37:31.033883 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1066: this message will repeat every 5th retry.
W20260810 22:37:32.104220 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1071: this message will repeat every 5th retry.
W20260810 22:37:32.596601 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 30 similar messages]
W20260810 22:37:33.396020 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1076: this message will repeat every 5th retry.
W20260810 22:37:33.502493 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1071: this message will repeat every 5th retry.
W20260810 22:37:34.496575 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1076: this message will repeat every 5th retry.
W20260810 22:37:35.883808 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1076: this message will repeat every 5th retry.
W20260810 22:37:36.010869 8579 heartbeater.cc:644] Failed to heartbeat to 127.8.18.190:40323 (420 consecutive failures): Network error: Failed to ping master at 127.8.18.190:40323: Client connection negotiation failed: client connection to 127.8.18.190:40323: connect: Connection refused (error 111)
W20260810 22:37:36.051148 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1081: this message will repeat every 5th retry.
W20260810 22:37:36.208123 8756 heartbeater.cc:644] Failed to heartbeat to 127.8.18.190:40323 (422 consecutive failures): Network error: Failed to ping master at 127.8.18.190:40323: Client connection negotiation failed: client connection to 127.8.18.190:40323: connect: Connection refused (error 111)
W20260810 22:37:37.047429 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1081: this message will repeat every 5th retry.
W20260810 22:37:37.756554 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 29 similar messages]
W20260810 22:37:38.311721 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1081: this message will repeat every 5th retry.
W20260810 22:37:38.662818 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1086: this message will repeat every 5th retry.
W20260810 22:37:39.407461 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1086: this message will repeat every 5th retry.
W20260810 22:37:40.663288 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1086: this message will repeat every 5th retry.
W20260810 22:37:41.181238 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1091: this message will repeat every 5th retry.
W20260810 22:37:42.090204 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1091: this message will repeat every 5th retry.
Master Summary
UUID | Address | Status
--------------------------------+--------------------+-------------
<unknown> (127.8.18.190:40323) | 127.8.18.190:40323 | UNAVAILABLE
Error from 127.8.18.190:40323: Network error: Client connection negotiation failed: client connection to 127.8.18.190:40323: connect: Connection refused (error 111) (UNAVAILABLE)
All reported replicas are:
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+----------+--------------+--------------+------------
Tablet Server Summary
Version Summary
Version | Servers
---------+---------
The cluster doesn't have any matching tablets
The cluster doesn't have any matching system tables
The cluster doesn't have any matching tables
==================
Warnings:
==================
master unusual flags check error: 1 of 1 masters were not available to retrieve unusual flags
master diverged flags check error: 1 of 1 masters were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from masters: failed to gather info from all masters: 1 of 1 had errors
Not found: master consensus error: no master consensus state available
Timed out: leader master liveness check error: Could not connect to the cluster: ConnectToClusterRpc(addrs: 127.8.18.190:40323, num_attempts: 344) passed its deadline: Network error: Client connection negotiation failed: client connection to 127.8.18.190:40323: connect: Connection refused (error 111)
FAILED
I20260810 22:37:43.040220 8266 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Timed out: leader master liveness check error: Could not connect to the cluster: ConnectToClusterRpc(addrs: 127.8.18.190:40323, num_attempts: 344) passed its deadline: Network error: Client connection negotiation failed: client connection to 127.8.18.190:40323: connect: Connection refused (error 111)
W20260810 22:37:43.054800 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 31 similar messages]
W20260810 22:37:43.055672 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1091: this message will repeat every 5th retry.
W20260810 22:37:43.636482 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1096: this message will repeat every 5th retry.
W20260810 22:37:44.507390 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1096: this message will repeat every 5th retry.
W20260810 22:37:45.803704 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1096: this message will repeat every 5th retry.
W20260810 22:37:46.126938 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1101: this message will repeat every 5th retry.
W20260810 22:37:47.337913 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1101: this message will repeat every 5th retry.
W20260810 22:37:48.100396 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 29 similar messages]
W20260810 22:37:48.101171 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1101: this message will repeat every 5th retry.
W20260810 22:37:48.919027 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1106: this message will repeat every 5th retry.
W20260810 22:37:49.932667 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1106: this message will repeat every 5th retry.
W20260810 22:37:50.690904 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1106: this message will repeat every 5th retry.
W20260810 22:37:51.479241 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1111: this message will repeat every 5th retry.
W20260810 22:37:52.294092 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1111: this message will repeat every 5th retry.
W20260810 22:37:53.284888 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 29 similar messages]
W20260810 22:37:53.285651 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1111: this message will repeat every 5th retry.
W20260810 22:37:53.957428 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1116: this message will repeat every 5th retry.
W20260810 22:37:54.663589 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1116: this message will repeat every 5th retry.
W20260810 22:37:56.079376 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1116: this message will repeat every 5th retry.
W20260810 22:37:56.641196 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1121: this message will repeat every 5th retry.
W20260810 22:37:57.404845 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1121: this message will repeat every 5th retry.
W20260810 22:37:58.301689 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 27 similar messages]
W20260810 22:37:58.788031 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1121: this message will repeat every 5th retry.
W20260810 22:37:59.411068 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1126: this message will repeat every 5th retry.
W20260810 22:37:59.840364 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1126: this message will repeat every 5th retry.
W20260810 22:38:01.326648 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1126: this message will repeat every 5th retry.
W20260810 22:38:02.070436 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1131: this message will repeat every 5th retry.
W20260810 22:38:02.284067 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1131: this message will repeat every 5th retry.
W20260810 22:38:03.311466 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 30 similar messages]
W20260810 22:38:03.900254 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1131: this message will repeat every 5th retry.
W20260810 22:38:04.778674 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1136: this message will repeat every 5th retry.
W20260810 22:38:04.778772 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1136: this message will repeat every 5th retry.
W20260810 22:38:06.105576 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1136: this message will repeat every 5th retry.
W20260810 22:38:07.138273 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1141: this message will repeat every 5th retry.
W20260810 22:38:07.417483 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1141: this message will repeat every 5th retry.
W20260810 22:38:08.540022 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 30 similar messages]
W20260810 22:38:08.754777 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1141: this message will repeat every 5th retry.
W20260810 22:38:09.685550 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1146: this message will repeat every 5th retry.
W20260810 22:38:09.804991 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1146: this message will repeat every 5th retry.
W20260810 22:38:10.847254 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1146: this message will repeat every 5th retry.
W20260810 22:38:12.175740 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1151: this message will repeat every 5th retry.
W20260810 22:38:12.385455 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1151: this message will repeat every 5th retry.
W20260810 22:38:13.181397 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1151: this message will repeat every 5th retry.
W20260810 22:38:13.693859 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 30 similar messages]
W20260810 22:38:14.665812 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1156: this message will repeat every 5th retry.
W20260810 22:38:14.724895 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1156: this message will repeat every 5th retry.
W20260810 22:38:15.851977 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1156: this message will repeat every 5th retry.
W20260810 22:38:17.110296 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1161: this message will repeat every 5th retry.
W20260810 22:38:17.362301 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1161: this message will repeat every 5th retry.
W20260810 22:38:18.508912 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1161: this message will repeat every 5th retry.
W20260810 22:38:18.755291 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 30 similar messages]
W20260810 22:38:19.189630 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1166: this message will repeat every 5th retry.
W20260810 22:38:19.721905 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1166: this message will repeat every 5th retry.
W20260810 22:38:20.765434 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1166: this message will repeat every 5th retry.
W20260810 22:38:21.609073 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1171: this message will repeat every 5th retry.
W20260810 22:38:22.038849 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1171: this message will repeat every 5th retry.
W20260810 22:38:23.290051 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1171: this message will repeat every 5th retry.
W20260810 22:38:23.804489 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 31 similar messages]
W20260810 22:38:24.236438 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1176: this message will repeat every 5th retry.
W20260810 22:38:24.745947 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1176: this message will repeat every 5th retry.
W20260810 22:38:25.833369 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1176: this message will repeat every 5th retry.
W20260810 22:38:26.926348 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1181: this message will repeat every 5th retry.
W20260810 22:38:27.132740 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1181: this message will repeat every 5th retry.
W20260810 22:38:28.303246 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1181: this message will repeat every 5th retry.
W20260810 22:38:28.968279 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 29 similar messages]
W20260810 22:38:29.469158 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1186: this message will repeat every 5th retry.
W20260810 22:38:29.703634 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1186: this message will repeat every 5th retry.
W20260810 22:38:30.725327 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1186: this message will repeat every 5th retry.
W20260810 22:38:31.898936 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1191: this message will repeat every 5th retry.
W20260810 22:38:31.920192 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1191: this message will repeat every 5th retry.
W20260810 22:38:33.280637 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1191: this message will repeat every 5th retry.
W20260810 22:38:34.052392 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 31 similar messages]
W20260810 22:38:34.053107 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1196: this message will repeat every 5th retry.
W20260810 22:38:34.511842 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1196: this message will repeat every 5th retry.
W20260810 22:38:36.091727 8579 heartbeater.cc:644] Failed to heartbeat to 127.8.18.190:40323 (480 consecutive failures): Network error: Failed to ping master at 127.8.18.190:40323: Client connection negotiation failed: client connection to 127.8.18.190:40323: connect: Connection refused (error 111)
W20260810 22:38:36.161613 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1196: this message will repeat every 5th retry.
W20260810 22:38:36.277657 8756 heartbeater.cc:644] Failed to heartbeat to 127.8.18.190:40323 (482 consecutive failures): Network error: Failed to ping master at 127.8.18.190:40323: Client connection negotiation failed: client connection to 127.8.18.190:40323: connect: Connection refused (error 111)
W20260810 22:38:36.620527 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1201: this message will repeat every 5th retry.
W20260810 22:38:37.243166 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1201: this message will repeat every 5th retry.
W20260810 22:38:38.975905 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1206: this message will repeat every 5th retry.
W20260810 22:38:38.976882 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1201: this message will repeat every 5th retry.
W20260810 22:38:39.124606 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 28 similar messages]
W20260810 22:38:39.631938 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1206: this message will repeat every 5th retry.
W20260810 22:38:41.101089 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1206: this message will repeat every 5th retry.
W20260810 22:38:41.405972 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1211: this message will repeat every 5th retry.
W20260810 22:38:42.072052 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1211: this message will repeat every 5th retry.
W20260810 22:38:43.510561 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1211: this message will repeat every 5th retry.
W20260810 22:38:44.046208 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1216: this message will repeat every 5th retry.
Master Summary
UUID | Address | Status
--------------------------------+--------------------+-------------
<unknown> (127.8.18.190:40323) | 127.8.18.190:40323 | UNAVAILABLE
Error from 127.8.18.190:40323: Network error: Client connection negotiation failed: client connection to 127.8.18.190:40323: connect: Connection refused (error 111) (UNAVAILABLE)
All reported replicas are:
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+----------+--------------+--------------+------------
Tablet Server Summary
Version Summary
Version | Servers
---------+---------
The cluster doesn't have any matching tablets
The cluster doesn't have any matching system tables
The cluster doesn't have any matching tables
==================
Warnings:
==================
master unusual flags check error: 1 of 1 masters were not available to retrieve unusual flags
master diverged flags check error: 1 of 1 masters were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from masters: failed to gather info from all masters: 1 of 1 had errors
Not found: master consensus error: no master consensus state available
Timed out: leader master liveness check error: Could not connect to the cluster: ConnectToClusterRpc(addrs: 127.8.18.190:40323, num_attempts: 344) passed its deadline: Network error: Client connection negotiation failed: client connection to 127.8.18.190:40323: connect: Connection refused (error 111)
FAILED
I20260810 22:38:44.248867 8266 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Timed out: leader master liveness check error: Could not connect to the cluster: ConnectToClusterRpc(addrs: 127.8.18.190:40323, num_attempts: 344) passed its deadline: Network error: Client connection negotiation failed: client connection to 127.8.18.190:40323: connect: Connection refused (error 111)
W20260810 22:38:44.481714 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 31 similar messages]
W20260810 22:38:44.482503 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1216: this message will repeat every 5th retry.
W20260810 22:38:45.896361 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1216: this message will repeat every 5th retry.
W20260810 22:38:46.571638 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1221: this message will repeat every 5th retry.
W20260810 22:38:46.797039 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1221: this message will repeat every 5th retry.
W20260810 22:38:48.308530 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1221: this message will repeat every 5th retry.
W20260810 22:38:48.789892 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1226: this message will repeat every 5th retry.
W20260810 22:38:49.345324 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1226: this message will repeat every 5th retry.
W20260810 22:38:49.754428 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 32 similar messages]
W20260810 22:38:50.895784 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1226: this message will repeat every 5th retry.
W20260810 22:38:51.286609 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1231: this message will repeat every 5th retry.
W20260810 22:38:51.780265 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1231: this message will repeat every 5th retry.
W20260810 22:38:53.486878 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1231: this message will repeat every 5th retry.
W20260810 22:38:53.682976 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1236: this message will repeat every 5th retry.
W20260810 22:38:54.414963 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1236: this message will repeat every 5th retry.
W20260810 22:38:54.974013 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 30 similar messages]
W20260810 22:38:56.063987 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1236: this message will repeat every 5th retry.
W20260810 22:38:56.064095 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1241: this message will repeat every 5th retry.
W20260810 22:38:56.950016 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1241: this message will repeat every 5th retry.
W20260810 22:38:58.364239 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1241: this message will repeat every 5th retry.
W20260810 22:38:58.594213 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1246: this message will repeat every 5th retry.
W20260810 22:38:59.182392 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1246: this message will repeat every 5th retry.
W20260810 22:38:59.979737 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 31 similar messages]
W20260810 22:39:00.472368 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1246: this message will repeat every 5th retry.
W20260810 22:39:01.110539 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1251: this message will repeat every 5th retry.
W20260810 22:39:01.675132 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1251: this message will repeat every 5th retry.
W20260810 22:39:03.160722 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1251: this message will repeat every 5th retry.
W20260810 22:39:03.666954 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1256: this message will repeat every 5th retry.
W20260810 22:39:04.272295 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1256: this message will repeat every 5th retry.
W20260810 22:39:05.192910 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 29 similar messages]
W20260810 22:39:05.686673 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1256: this message will repeat every 5th retry.
W20260810 22:39:06.096130 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1261: this message will repeat every 5th retry.
W20260810 22:39:06.949941 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1261: this message will repeat every 5th retry.
W20260810 22:39:08.372686 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1261: this message will repeat every 5th retry.
W20260810 22:39:08.377743 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1266: this message will repeat every 5th retry.
W20260810 22:39:09.409883 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1266: this message will repeat every 5th retry.
W20260810 22:39:10.306778 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 31 similar messages]
W20260810 22:39:10.726263 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1271: this message will repeat every 5th retry.
W20260810 22:39:10.817013 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1266: this message will repeat every 5th retry.
W20260810 22:39:11.914481 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1271: this message will repeat every 5th retry.
W20260810 22:39:13.188071 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1276: this message will repeat every 5th retry.
W20260810 22:39:13.251060 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1271: this message will repeat every 5th retry.
W20260810 22:39:14.468523 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1276: this message will repeat every 5th retry.
W20260810 22:39:15.316417 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 28 similar messages]
W20260810 22:39:15.883359 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1276: this message will repeat every 5th retry.
W20260810 22:39:15.940609 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1281: this message will repeat every 5th retry.
W20260810 22:39:17.092021 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1281: this message will repeat every 5th retry.
W20260810 22:39:18.595292 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1286: this message will repeat every 5th retry.
W20260810 22:39:18.870039 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1281: this message will repeat every 5th retry.
W20260810 22:39:19.487089 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1286: this message will repeat every 5th retry.
W20260810 22:39:20.372126 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 28 similar messages]
W20260810 22:39:20.825723 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1291: this message will repeat every 5th retry.
W20260810 22:39:21.551777 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1286: this message will repeat every 5th retry.
W20260810 22:39:22.213862 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1291: this message will repeat every 5th retry.
W20260810 22:39:23.230923 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1296: this message will repeat every 5th retry.
W20260810 22:39:24.290089 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1291: this message will repeat every 5th retry.
W20260810 22:39:24.637503 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1296: this message will repeat every 5th retry.
W20260810 22:39:25.428043 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 30 similar messages]
W20260810 22:39:25.948467 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1301: this message will repeat every 5th retry.
W20260810 22:39:26.633724 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1301: this message will repeat every 5th retry.
W20260810 22:39:26.883157 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1296: this message will repeat every 5th retry.
W20260810 22:39:28.422937 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1306: this message will repeat every 5th retry.
W20260810 22:39:29.174914 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1301: this message will repeat every 5th retry.
W20260810 22:39:29.381843 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1306: this message will repeat every 5th retry.
W20260810 22:39:30.449815 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 28 similar messages]
W20260810 22:39:31.142623 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1311: this message will repeat every 5th retry.
W20260810 22:39:31.884238 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1311: this message will repeat every 5th retry.
W20260810 22:39:32.034862 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1306: this message will repeat every 5th retry.
W20260810 22:39:33.340487 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1316: this message will repeat every 5th retry.
W20260810 22:39:34.353461 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1316: this message will repeat every 5th retry.
W20260810 22:39:34.388288 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1311: this message will repeat every 5th retry.
W20260810 22:39:35.520790 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 30 similar messages]
W20260810 22:39:35.796449 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1321: this message will repeat every 5th retry.
W20260810 22:39:36.173853 8579 heartbeater.cc:644] Failed to heartbeat to 127.8.18.190:40323 (540 consecutive failures): Network error: Failed to ping master at 127.8.18.190:40323: Client connection negotiation failed: client connection to 127.8.18.190:40323: connect: Connection refused (error 111)
W20260810 22:39:36.346647 8756 heartbeater.cc:644] Failed to heartbeat to 127.8.18.190:40323 (542 consecutive failures): Network error: Failed to ping master at 127.8.18.190:40323: Client connection negotiation failed: client connection to 127.8.18.190:40323: connect: Connection refused (error 111)
W20260810 22:39:36.796197 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1321: this message will repeat every 5th retry.
W20260810 22:39:36.813572 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1316: this message will repeat every 5th retry.
W20260810 22:39:38.090862 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1326: this message will repeat every 5th retry.
W20260810 22:39:39.173686 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1321: this message will repeat every 5th retry.
W20260810 22:39:39.203379 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1326: this message will repeat every 5th retry.
W20260810 22:39:40.427045 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1331: this message will repeat every 5th retry.
W20260810 22:39:40.543105 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 31 similar messages]
W20260810 22:39:41.557025 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1331: this message will repeat every 5th retry.
W20260810 22:39:41.702986 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1326: this message will repeat every 5th retry.
W20260810 22:39:42.604374 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1336: this message will repeat every 5th retry.
W20260810 22:39:43.999017 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1336: this message will repeat every 5th retry.
W20260810 22:39:44.450174 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1331: this message will repeat every 5th retry.
W20260810 22:39:45.246842 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1341: this message will repeat every 5th retry.
Master Summary
UUID | Address | Status
--------------------------------+--------------------+-------------
<unknown> (127.8.18.190:40323) | 127.8.18.190:40323 | UNAVAILABLE
Error from 127.8.18.190:40323: Network error: Client connection negotiation failed: client connection to 127.8.18.190:40323: connect: Connection refused (error 111) (UNAVAILABLE)
All reported replicas are:
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+----------+--------------+--------------+------------
Tablet Server Summary
Version Summary
Version | Servers
---------+---------
The cluster doesn't have any matching tablets
The cluster doesn't have any matching system tables
The cluster doesn't have any matching tables
==================
Warnings:
==================
master unusual flags check error: 1 of 1 masters were not available to retrieve unusual flags
master diverged flags check error: 1 of 1 masters were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from masters: failed to gather info from all masters: 1 of 1 had errors
Not found: master consensus error: no master consensus state available
Timed out: leader master liveness check error: Could not connect to the cluster: ConnectToClusterRpc(addrs: 127.8.18.190:40323, num_attempts: 344) passed its deadline: Network error: Client connection negotiation failed: client connection to 127.8.18.190:40323: connect: Connection refused (error 111)
FAILED
I20260810 22:39:45.427935 8266 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Timed out: leader master liveness check error: Could not connect to the cluster: ConnectToClusterRpc(addrs: 127.8.18.190:40323, num_attempts: 344) passed its deadline: Network error: Client connection negotiation failed: client connection to 127.8.18.190:40323: connect: Connection refused (error 111)
W20260810 22:39:45.670593 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 29 similar messages]
W20260810 22:39:46.622054 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1336: this message will repeat every 5th retry.
W20260810 22:39:46.677400 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1341: this message will repeat every 5th retry.
W20260810 22:39:47.741019 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1346: this message will repeat every 5th retry.
W20260810 22:39:49.204466 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1341: this message will repeat every 5th retry.
W20260810 22:39:49.397270 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1346: this message will repeat every 5th retry.
W20260810 22:39:50.331087 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1351: this message will repeat every 5th retry.
W20260810 22:39:50.814502 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 29 similar messages]
W20260810 22:39:51.784029 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1346: this message will repeat every 5th retry.
W20260810 22:39:51.812534 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1351: this message will repeat every 5th retry.
W20260810 22:39:53.044257 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1356: this message will repeat every 5th retry.
W20260810 22:39:54.176599 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1351: this message will repeat every 5th retry.
W20260810 22:39:54.595582 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1356: this message will repeat every 5th retry.
W20260810 22:39:55.696499 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1361: this message will repeat every 5th retry.
W20260810 22:39:56.165443 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 30 similar messages]
W20260810 22:39:56.781224 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1356: this message will repeat every 5th retry.
W20260810 22:39:57.297317 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1361: this message will repeat every 5th retry.
W20260810 22:39:58.300640 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1366: this message will repeat every 5th retry.
W20260810 22:39:59.437206 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1361: this message will repeat every 5th retry.
W20260810 22:39:59.832181 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1366: this message will repeat every 5th retry.
W20260810 22:40:01.095106 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1371: this message will repeat every 5th retry.
W20260810 22:40:01.342659 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 29 similar messages]
W20260810 22:40:02.203586 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1366: this message will repeat every 5th retry.
W20260810 22:40:02.347688 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1371: this message will repeat every 5th retry.
W20260810 22:40:03.351060 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1376: this message will repeat every 5th retry.
W20260810 22:40:04.657907 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1371: this message will repeat every 5th retry.
W20260810 22:40:04.790292 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1376: this message will repeat every 5th retry.
W20260810 22:40:05.856612 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1381: this message will repeat every 5th retry.
W20260810 22:40:06.376593 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 30 similar messages]
W20260810 22:40:06.920400 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1376: this message will repeat every 5th retry.
W20260810 22:40:07.380264 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1381: this message will repeat every 5th retry.
W20260810 22:40:08.290201 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1386: this message will repeat every 5th retry.
W20260810 22:40:09.364296 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1381: this message will repeat every 5th retry.
W20260810 22:40:10.136591 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1386: this message will repeat every 5th retry.
W20260810 22:40:10.806080 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1391: this message will repeat every 5th retry.
W20260810 22:40:11.489037 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 29 similar messages]
W20260810 22:40:12.064986 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1386: this message will repeat every 5th retry.
W20260810 22:40:12.481753 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1391: this message will repeat every 5th retry.
W20260810 22:40:13.508436 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1396: this message will repeat every 5th retry.
W20260810 22:40:14.584944 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1391: this message will repeat every 5th retry.
W20260810 22:40:14.880970 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1396: this message will repeat every 5th retry.
W20260810 22:40:16.028185 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1401: this message will repeat every 5th retry.
W20260810 22:40:16.513967 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 30 similar messages]
W20260810 22:40:16.957916 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1396: this message will repeat every 5th retry.
W20260810 22:40:17.158466 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1401: this message will repeat every 5th retry.
W20260810 22:40:18.440323 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1406: this message will repeat every 5th retry.
W20260810 22:40:19.355986 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1401: this message will repeat every 5th retry.
W20260810 22:40:19.474177 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1406: this message will repeat every 5th retry.
W20260810 22:40:20.943754 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1411: this message will repeat every 5th retry.
W20260810 22:40:21.720717 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 31 similar messages]
W20260810 22:40:21.721493 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1411: this message will repeat every 5th retry.
W20260810 22:40:21.930275 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1406: this message will repeat every 5th retry.
W20260810 22:40:23.616451 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1416: this message will repeat every 5th retry.
W20260810 22:40:24.325532 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1416: this message will repeat every 5th retry.
W20260810 22:40:24.523614 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1411: this message will repeat every 5th retry.
W20260810 22:40:26.101980 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1421: this message will repeat every 5th retry.
W20260810 22:40:26.474016 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1421: this message will repeat every 5th retry.
W20260810 22:40:26.796911 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 30 similar messages]
W20260810 22:40:26.797761 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1416: this message will repeat every 5th retry.
W20260810 22:40:28.692412 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1426: this message will repeat every 5th retry.
W20260810 22:40:29.046255 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1426: this message will repeat every 5th retry.
W20260810 22:40:29.591511 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1421: this message will repeat every 5th retry.
W20260810 22:40:31.024550 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1431: this message will repeat every 5th retry.
W20260810 22:40:31.458765 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1431: this message will repeat every 5th retry.
W20260810 22:40:31.945953 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 29 similar messages]
W20260810 22:40:31.946730 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1426: this message will repeat every 5th retry.
W20260810 22:40:33.258698 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1436: this message will repeat every 5th retry.
W20260810 22:40:34.174136 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1436: this message will repeat every 5th retry.
W20260810 22:40:34.451292 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1431: this message will repeat every 5th retry.
W20260810 22:40:35.634313 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1441: this message will repeat every 5th retry.
W20260810 22:40:36.251628 8579 heartbeater.cc:644] Failed to heartbeat to 127.8.18.190:40323 (600 consecutive failures): Network error: Failed to ping master at 127.8.18.190:40323: Client connection negotiation failed: client connection to 127.8.18.190:40323: connect: Connection refused (error 111)
W20260810 22:40:36.416184 8756 heartbeater.cc:644] Failed to heartbeat to 127.8.18.190:40323 (602 consecutive failures): Network error: Failed to ping master at 127.8.18.190:40323: Client connection negotiation failed: client connection to 127.8.18.190:40323: connect: Connection refused (error 111)
W20260810 22:40:36.722223 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1441: this message will repeat every 5th retry.
W20260810 22:40:37.010227 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 30 similar messages]
W20260810 22:40:37.011081 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1436: this message will repeat every 5th retry.
W20260810 22:40:37.877815 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1446: this message will repeat every 5th retry.
W20260810 22:40:39.389899 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1446: this message will repeat every 5th retry.
W20260810 22:40:39.567605 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1441: this message will repeat every 5th retry.
W20260810 22:40:40.432912 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1451: this message will repeat every 5th retry.
W20260810 22:40:41.902878 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1451: this message will repeat every 5th retry.
W20260810 22:40:42.047098 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 29 similar messages]
W20260810 22:40:42.082427 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1446: this message will repeat every 5th retry.
W20260810 22:40:43.087273 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1456: this message will repeat every 5th retry.
W20260810 22:40:44.325135 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1456: this message will repeat every 5th retry.
W20260810 22:40:44.547066 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1451: this message will repeat every 5th retry.
W20260810 22:40:45.738688 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1461: this message will repeat every 5th retry.
Master Summary
UUID | Address | Status
--------------------------------+--------------------+-------------
<unknown> (127.8.18.190:40323) | 127.8.18.190:40323 | UNAVAILABLE
Error from 127.8.18.190:40323: Network error: Client connection negotiation failed: client connection to 127.8.18.190:40323: connect: Connection refused (error 111) (UNAVAILABLE)
All reported replicas are:
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+----------+--------------+--------------+------------
Tablet Server Summary
Version Summary
Version | Servers
---------+---------
The cluster doesn't have any matching tablets
The cluster doesn't have any matching system tables
The cluster doesn't have any matching tables
==================
Warnings:
==================
master unusual flags check error: 1 of 1 masters were not available to retrieve unusual flags
master diverged flags check error: 1 of 1 masters were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from masters: failed to gather info from all masters: 1 of 1 had errors
Not found: master consensus error: no master consensus state available
Timed out: leader master liveness check error: Could not connect to the cluster: ConnectToClusterRpc(addrs: 127.8.18.190:40323, num_attempts: 344) passed its deadline: Network error: Client connection negotiation failed: client connection to 127.8.18.190:40323: connect: Connection refused (error 111)
FAILED
I20260810 22:40:46.642951 8266 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Timed out: leader master liveness check error: Could not connect to the cluster: ConnectToClusterRpc(addrs: 127.8.18.190:40323, num_attempts: 344) passed its deadline: Network error: Client connection negotiation failed: client connection to 127.8.18.190:40323: connect: Connection refused (error 111)
W20260810 22:40:46.810010 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1461: this message will repeat every 5th retry.
W20260810 22:40:46.923097 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1456: this message will repeat every 5th retry.
W20260810 22:40:47.287464 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 30 similar messages]
W20260810 22:40:48.378530 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1466: this message will repeat every 5th retry.
W20260810 22:40:49.268688 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1461: this message will repeat every 5th retry.
W20260810 22:40:49.522166 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1466: this message will repeat every 5th retry.
W20260810 22:40:50.995285 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1471: this message will repeat every 5th retry.
W20260810 22:40:52.100492 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1466: this message will repeat every 5th retry.
W20260810 22:40:52.326365 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 28 similar messages]
W20260810 22:40:52.327138 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1471: this message will repeat every 5th retry.
W20260810 22:40:53.375339 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1476: this message will repeat every 5th retry.
W20260810 22:40:54.443104 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1471: this message will repeat every 5th retry.
W20260810 22:40:55.063951 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1476: this message will repeat every 5th retry.
W20260810 22:40:55.832129 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1481: this message will repeat every 5th retry.
W20260810 22:40:56.784987 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1476: this message will repeat every 5th retry.
W20260810 22:40:57.356055 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 30 similar messages]
W20260810 22:40:57.614955 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1481: this message will repeat every 5th retry.
W20260810 22:40:58.306851 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1486: this message will repeat every 5th retry.
W20260810 22:40:59.292788 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1481: this message will repeat every 5th retry.
W20260810 22:41:00.064761 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1486: this message will repeat every 5th retry.
W20260810 22:41:00.818396 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1491: this message will repeat every 5th retry.
W20260810 22:41:02.008001 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1486: this message will repeat every 5th retry.
W20260810 22:41:02.384841 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 29 similar messages]
W20260810 22:41:02.454456 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1491: this message will repeat every 5th retry.
W20260810 22:41:03.073522 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1496: this message will repeat every 5th retry.
W20260810 22:41:04.242060 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1491: this message will repeat every 5th retry.
W20260810 22:41:04.880681 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1496: this message will repeat every 5th retry.
W20260810 22:41:05.546507 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1501: this message will repeat every 5th retry.
W20260810 22:41:06.949638 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1496: this message will repeat every 5th retry.
W20260810 22:41:07.294773 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1501: this message will repeat every 5th retry.
W20260810 22:41:07.509550 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 30 similar messages]
W20260810 22:41:08.076864 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1506: this message will repeat every 5th retry.
W20260810 22:41:09.669625 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1501: this message will repeat every 5th retry.
W20260810 22:41:09.710949 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1506: this message will repeat every 5th retry.
W20260810 22:41:10.795562 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1511: this message will repeat every 5th retry.
W20260810 22:41:12.174199 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1511: this message will repeat every 5th retry.
W20260810 22:41:12.179908 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1506: this message will repeat every 5th retry.
W20260810 22:41:12.561502 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 28 similar messages]
W20260810 22:41:13.383411 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1516: this message will repeat every 5th retry.
W20260810 22:41:14.481051 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1511: this message will repeat every 5th retry.
W20260810 22:41:14.766882 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1516: this message will repeat every 5th retry.
W20260810 22:41:15.711062 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1521: this message will repeat every 5th retry.
W20260810 22:41:16.747447 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1516: this message will repeat every 5th retry.
W20260810 22:41:17.364431 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1521: this message will repeat every 5th retry.
W20260810 22:41:17.723788 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 31 similar messages]
W20260810 22:41:18.312508 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1526: this message will repeat every 5th retry.
W20260810 22:41:19.458820 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1521: this message will repeat every 5th retry.
W20260810 22:41:19.983888 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1526: this message will repeat every 5th retry.
W20260810 22:41:20.763749 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1531: this message will repeat every 5th retry.
W20260810 22:41:22.307464 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1526: this message will repeat every 5th retry.
W20260810 22:41:22.443915 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1531: this message will repeat every 5th retry.
W20260810 22:41:23.024947 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 30 similar messages]
W20260810 22:41:23.288538 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1536: this message will repeat every 5th retry.
W20260810 22:41:24.780921 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1531: this message will repeat every 5th retry.
W20260810 22:41:24.849810 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1536: this message will repeat every 5th retry.
W20260810 22:41:25.617319 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1541: this message will repeat every 5th retry.
W20260810 22:41:27.329936 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1536: this message will repeat every 5th retry.
W20260810 22:41:27.504228 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1541: this message will repeat every 5th retry.
W20260810 22:41:27.990935 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1546: this message will repeat every 5th retry.
W20260810 22:41:28.044661 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 30 similar messages]
W20260810 22:41:30.059763 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1541: this message will repeat every 5th retry.
W20260810 22:41:30.095973 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1546: this message will repeat every 5th retry.
W20260810 22:41:30.250969 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1551: this message will repeat every 5th retry.
W20260810 22:41:32.596290 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1551: this message will repeat every 5th retry.
W20260810 22:41:32.772517 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1546: this message will repeat every 5th retry.
W20260810 22:41:32.794648 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1556: this message will repeat every 5th retry.
W20260810 22:41:33.047609 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 28 similar messages]
W20260810 22:41:35.221120 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1556: this message will repeat every 5th retry.
W20260810 22:41:35.260617 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1561: this message will repeat every 5th retry.
W20260810 22:41:35.281438 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1551: this message will repeat every 5th retry.
W20260810 22:41:36.328256 8579 heartbeater.cc:644] Failed to heartbeat to 127.8.18.190:40323 (660 consecutive failures): Network error: Failed to ping master at 127.8.18.190:40323: Client connection negotiation failed: client connection to 127.8.18.190:40323: connect: Connection refused (error 111)
W20260810 22:41:36.484838 8756 heartbeater.cc:644] Failed to heartbeat to 127.8.18.190:40323 (662 consecutive failures): Network error: Failed to ping master at 127.8.18.190:40323: Client connection negotiation failed: client connection to 127.8.18.190:40323: connect: Connection refused (error 111)
W20260810 22:41:37.606619 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1561: this message will repeat every 5th retry.
W20260810 22:41:37.611727 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1556: this message will repeat every 5th retry.
W20260810 22:41:37.876662 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1566: this message will repeat every 5th retry.
W20260810 22:41:38.159015 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 29 similar messages]
W20260810 22:41:40.150276 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1566: this message will repeat every 5th retry.
W20260810 22:41:40.312948 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1561: this message will repeat every 5th retry.
W20260810 22:41:40.447430 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1571: this message will repeat every 5th retry.
W20260810 22:41:42.507081 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1571: this message will repeat every 5th retry.
W20260810 22:41:42.901192 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1566: this message will repeat every 5th retry.
W20260810 22:41:42.928911 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1576: this message will repeat every 5th retry.
W20260810 22:41:43.371322 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 30 similar messages]
W20260810 22:41:44.976961 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1576: this message will repeat every 5th retry.
W20260810 22:41:45.432744 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1581: this message will repeat every 5th retry.
W20260810 22:41:45.448942 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1571: this message will repeat every 5th retry.
W20260810 22:41:47.100930 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1581: this message will repeat every 5th retry.
W20260810 22:41:47.683139 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1576: this message will repeat every 5th retry.
Master Summary
UUID | Address | Status
--------------------------------+--------------------+-------------
<unknown> (127.8.18.190:40323) | 127.8.18.190:40323 | UNAVAILABLE
Error from 127.8.18.190:40323: Network error: Client connection negotiation failed: client connection to 127.8.18.190:40323: connect: Connection refused (error 111) (UNAVAILABLE)
All reported replicas are:
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+----------+--------------+--------------+------------
Tablet Server Summary
Version Summary
Version | Servers
---------+---------
The cluster doesn't have any matching tablets
The cluster doesn't have any matching system tables
The cluster doesn't have any matching tables
==================
Warnings:
==================
master unusual flags check error: 1 of 1 masters were not available to retrieve unusual flags
master diverged flags check error: 1 of 1 masters were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from masters: failed to gather info from all masters: 1 of 1 had errors
Not found: master consensus error: no master consensus state available
Timed out: leader master liveness check error: Could not connect to the cluster: ConnectToClusterRpc(addrs: 127.8.18.190:40323, num_attempts: 344) passed its deadline: Network error: Client connection negotiation failed: client connection to 127.8.18.190:40323: connect: Connection refused (error 111)
FAILED
I20260810 22:41:47.846485 8266 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Timed out: leader master liveness check error: Could not connect to the cluster: ConnectToClusterRpc(addrs: 127.8.18.190:40323, num_attempts: 344) passed its deadline: Network error: Client connection negotiation failed: client connection to 127.8.18.190:40323: connect: Connection refused (error 111)
W20260810 22:41:48.000079 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1586: this message will repeat every 5th retry.
W20260810 22:41:48.615671 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 31 similar messages]
W20260810 22:41:49.614319 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1586: this message will repeat every 5th retry.
W20260810 22:41:50.060668 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1581: this message will repeat every 5th retry.
W20260810 22:41:50.393165 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1591: this message will repeat every 5th retry.
W20260810 22:41:52.232383 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1591: this message will repeat every 5th retry.
W20260810 22:41:52.810530 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1586: this message will repeat every 5th retry.
W20260810 22:41:53.063664 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1596: this message will repeat every 5th retry.
W20260810 22:41:53.623229 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 30 similar messages]
W20260810 22:41:54.526917 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1596: this message will repeat every 5th retry.
W20260810 22:41:55.016633 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1591: this message will repeat every 5th retry.
W20260810 22:41:55.455058 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1601: this message will repeat every 5th retry.
W20260810 22:41:57.009965 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1601: this message will repeat every 5th retry.
W20260810 22:41:57.779017 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1606: this message will repeat every 5th retry.
W20260810 22:41:57.833664 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1596: this message will repeat every 5th retry.
W20260810 22:41:58.704717 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 30 similar messages]
W20260810 22:41:59.379293 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1606: this message will repeat every 5th retry.
W20260810 22:42:00.114264 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1611: this message will repeat every 5th retry.
W20260810 22:42:00.199474 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1601: this message will repeat every 5th retry.
W20260810 22:42:02.042194 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1611: this message will repeat every 5th retry.
W20260810 22:42:02.476796 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1616: this message will repeat every 5th retry.
W20260810 22:42:02.730746 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1606: this message will repeat every 5th retry.
W20260810 22:42:03.939113 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 31 similar messages]
W20260810 22:42:04.541435 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1616: this message will repeat every 5th retry.
W20260810 22:42:05.043717 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1621: this message will repeat every 5th retry.
W20260810 22:42:05.072201 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1611: this message will repeat every 5th retry.
W20260810 22:42:06.914222 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1621: this message will repeat every 5th retry.
W20260810 22:42:07.431360 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1626: this message will repeat every 5th retry.
W20260810 22:42:07.606956 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1616: this message will repeat every 5th retry.
W20260810 22:42:08.979851 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 29 similar messages]
W20260810 22:42:09.412096 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1626: this message will repeat every 5th retry.
W20260810 22:42:09.994578 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1621: this message will repeat every 5th retry.
W20260810 22:42:10.025134 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1631: this message will repeat every 5th retry.
W20260810 22:42:11.866578 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1631: this message will repeat every 5th retry.
W20260810 22:42:12.267540 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1626: this message will repeat every 5th retry.
W20260810 22:42:12.673270 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1636: this message will repeat every 5th retry.
W20260810 22:42:14.011688 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 31 similar messages]
W20260810 22:42:14.151129 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1636: this message will repeat every 5th retry.
W20260810 22:42:14.872547 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1631: this message will repeat every 5th retry.
W20260810 22:42:15.024616 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1641: this message will repeat every 5th retry.
W20260810 22:42:16.563328 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1641: this message will repeat every 5th retry.
W20260810 22:42:17.311367 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1636: this message will repeat every 5th retry.
W20260810 22:42:17.581806 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1646: this message will repeat every 5th retry.
W20260810 22:42:19.034813 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 29 similar messages]
W20260810 22:42:19.035597 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1646: this message will repeat every 5th retry.
W20260810 22:42:19.675052 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1641: this message will repeat every 5th retry.
W20260810 22:42:20.167518 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1651: this message will repeat every 5th retry.
W20260810 22:42:21.568269 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1651: this message will repeat every 5th retry.
W20260810 22:42:22.267349 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1646: this message will repeat every 5th retry.
W20260810 22:42:22.471019 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1656: this message will repeat every 5th retry.
W20260810 22:42:24.050614 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 30 similar messages]
W20260810 22:42:24.051437 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1656: this message will repeat every 5th retry.
W20260810 22:42:24.555317 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1651: this message will repeat every 5th retry.
W20260810 22:42:24.814497 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1661: this message will repeat every 5th retry.
W20260810 22:42:26.592780 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1661: this message will repeat every 5th retry.
W20260810 22:42:27.231272 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1656: this message will repeat every 5th retry.
W20260810 22:42:27.265204 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1666: this message will repeat every 5th retry.
W20260810 22:42:29.055899 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 29 similar messages]
W20260810 22:42:29.198419 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1666: this message will repeat every 5th retry.
W20260810 22:42:29.477124 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1671: this message will repeat every 5th retry.
W20260810 22:42:29.567531 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1661: this message will repeat every 5th retry.
W20260810 22:42:31.849839 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1671: this message will repeat every 5th retry.
W20260810 22:42:31.899034 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1676: this message will repeat every 5th retry.
W20260810 22:42:32.377871 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1666: this message will repeat every 5th retry.
W20260810 22:42:34.202700 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 30 similar messages]
W20260810 22:42:34.203466 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1681: this message will repeat every 5th retry.
W20260810 22:42:34.359254 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1676: this message will repeat every 5th retry.
W20260810 22:42:34.988269 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1671: this message will repeat every 5th retry.
W20260810 22:42:36.409163 8579 heartbeater.cc:644] Failed to heartbeat to 127.8.18.190:40323 (720 consecutive failures): Network error: Failed to ping master at 127.8.18.190:40323: Client connection negotiation failed: client connection to 127.8.18.190:40323: connect: Connection refused (error 111)
W20260810 22:42:36.553666 8756 heartbeater.cc:644] Failed to heartbeat to 127.8.18.190:40323 (722 consecutive failures): Network error: Failed to ping master at 127.8.18.190:40323: Client connection negotiation failed: client connection to 127.8.18.190:40323: connect: Connection refused (error 111)
W20260810 22:42:36.870739 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1686: this message will repeat every 5th retry.
W20260810 22:42:37.158370 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1681: this message will repeat every 5th retry.
W20260810 22:42:37.613308 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1676: this message will repeat every 5th retry.
W20260810 22:42:39.263659 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 28 similar messages]
W20260810 22:42:39.468465 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1691: this message will repeat every 5th retry.
W20260810 22:42:39.674316 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1686: this message will repeat every 5th retry.
W20260810 22:42:39.885707 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1681: this message will repeat every 5th retry.
W20260810 22:42:41.944645 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1696: this message will repeat every 5th retry.
W20260810 22:42:42.302492 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1686: this message will repeat every 5th retry.
W20260810 22:42:42.339231 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1691: this message will repeat every 5th retry.
W20260810 22:42:44.476224 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 31 similar messages]
W20260810 22:42:44.476992 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1691: this message will repeat every 5th retry.
W20260810 22:42:44.627213 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1701: this message will repeat every 5th retry.
W20260810 22:42:44.776876 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1696: this message will repeat every 5th retry.
W20260810 22:42:47.091845 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1706: this message will repeat every 5th retry.
W20260810 22:42:47.205364 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1696: this message will repeat every 5th retry.
W20260810 22:42:47.205928 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1701: this message will repeat every 5th retry.
Master Summary
UUID | Address | Status
--------------------------------+--------------------+-------------
<unknown> (127.8.18.190:40323) | 127.8.18.190:40323 | UNAVAILABLE
Error from 127.8.18.190:40323: Network error: Client connection negotiation failed: client connection to 127.8.18.190:40323: connect: Connection refused (error 111) (UNAVAILABLE)
All reported replicas are:
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+----------+--------------+--------------+------------
Tablet Server Summary
Version Summary
Version | Servers
---------+---------
The cluster doesn't have any matching tablets
The cluster doesn't have any matching system tables
The cluster doesn't have any matching tables
==================
Warnings:
==================
master unusual flags check error: 1 of 1 masters were not available to retrieve unusual flags
master diverged flags check error: 1 of 1 masters were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from masters: failed to gather info from all masters: 1 of 1 had errors
Not found: master consensus error: no master consensus state available
Timed out: leader master liveness check error: Could not connect to the cluster: ConnectToClusterRpc(addrs: 127.8.18.190:40323, num_attempts: 344) passed its deadline: Network error: Client connection negotiation failed: client connection to 127.8.18.190:40323: connect: Connection refused (error 111)
FAILED
I20260810 22:42:48.980862 8266 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Timed out: leader master liveness check error: Could not connect to the cluster: ConnectToClusterRpc(addrs: 127.8.18.190:40323, num_attempts: 344) passed its deadline: Network error: Client connection negotiation failed: client connection to 127.8.18.190:40323: connect: Connection refused (error 111)
W20260810 22:42:49.586102 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 29 similar messages]
W20260810 22:42:49.586908 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1706: this message will repeat every 5th retry.
W20260810 22:42:49.753301 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1711: this message will repeat every 5th retry.
W20260810 22:42:50.035245 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1701: this message will repeat every 5th retry.
W20260810 22:42:51.972257 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1711: this message will repeat every 5th retry.
W20260810 22:42:52.298744 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1716: this message will repeat every 5th retry.
W20260810 22:42:52.654654 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1706: this message will repeat every 5th retry.
W20260810 22:42:54.491830 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1721: this message will repeat every 5th retry.
W20260810 22:42:54.566071 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1716: this message will repeat every 5th retry.
W20260810 22:42:54.672331 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 30 similar messages]
W20260810 22:42:55.228809 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1711: this message will repeat every 5th retry.
W20260810 22:42:57.036190 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1726: this message will repeat every 5th retry.
W20260810 22:42:57.075637 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1721: this message will repeat every 5th retry.
W20260810 22:42:57.722422 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1716: this message will repeat every 5th retry.
W20260810 22:42:59.381844 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1731: this message will repeat every 5th retry.
W20260810 22:42:59.586323 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1726: this message will repeat every 5th retry.
W20260810 22:42:59.847790 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 29 similar messages]
W20260810 22:43:00.444106 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1721: this message will repeat every 5th retry.
W20260810 22:43:01.725818 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1736: this message will repeat every 5th retry.
W20260810 22:43:01.986169 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1731: this message will repeat every 5th retry.
W20260810 22:43:03.025067 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1726: this message will repeat every 5th retry.
W20260810 22:43:04.235268 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1741: this message will repeat every 5th retry.
W20260810 22:43:04.317198 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1736: this message will repeat every 5th retry.
************************ BEGIN STACKS **************************
W20260810 22:43:04.899317 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 31 similar messages]
[New LWP 8271]
[New LWP 8272]
[New LWP 8274]
[New LWP 8936]
[New LWP 8937]
[New LWP 8938]
[New LWP 8940]
[New LWP 14525]
[New LWP 14526]
[New LWP 14527]
[New LWP 14528]
[New LWP 14534]
[New LWP 14535]
[New LWP 14536]
[New LWP 14537]
[New LWP 14539]
W20260810 22:43:05.437762 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1731: this message will repeat every 5th retry.
W20260810 22:43:06.632040 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1746: this message will repeat every 5th retry.
W20260810 22:43:06.991117 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1741: this message will repeat every 5th retry.
W20260810 22:43:07.810436 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1736: this message will repeat every 5th retry.
W20260810 22:43:09.084658 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1751: this message will repeat every 5th retry.
W20260810 22:43:09.618960 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1746: this message will repeat every 5th retry.
W20260810 22:43:10.090998 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 29 similar messages]
W20260810 22:43:10.416141 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1741: this message will repeat every 5th retry.
W20260810 22:43:11.530449 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1756: this message will repeat every 5th retry.
W20260810 22:43:12.239997 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1751: this message will repeat every 5th retry.
W20260810 22:43:12.998219 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1746: this message will repeat every 5th retry.
W20260810 22:43:13.899720 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1761: this message will repeat every 5th retry.
W20260810 22:43:14.437181 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1756: this message will repeat every 5th retry.
W20260810 22:43:15.388554 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 31 similar messages]
W20260810 22:43:15.471895 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1751: this message will repeat every 5th retry.
W20260810 22:43:16.538602 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1766: this message will repeat every 5th retry.
W20260810 22:43:16.941205 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1761: this message will repeat every 5th retry.
W20260810 22:43:17.975023 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1756: this message will repeat every 5th retry.
W20260810 22:43:18.920055 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1771: this message will repeat every 5th retry.
W20260810 22:43:19.423054 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1766: this message will repeat every 5th retry.
W20260810 22:43:20.189327 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1761: this message will repeat every 5th retry.
W20260810 22:43:20.438334 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 30 similar messages]
W20260810 22:43:21.617184 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1776: this message will repeat every 5th retry.
W20260810 22:43:22.150635 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1771: this message will repeat every 5th retry.
W20260810 22:43:22.594408 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1766: this message will repeat every 5th retry.
W20260810 22:43:23.826459 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1781: this message will repeat every 5th retry.
W20260810 22:43:24.873690 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1776: this message will repeat every 5th retry.
W20260810 22:43:25.307708 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1771: this message will repeat every 5th retry.
W20260810 22:43:25.647737 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 30 similar messages]
W20260810 22:43:26.125622 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1786: this message will repeat every 5th retry.
W20260810 22:43:27.625121 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1781: this message will repeat every 5th retry.
W20260810 22:43:27.843415 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1776: this message will repeat every 5th retry.
[Thread debugging using libthread_db enabled]
Using host libthread_db library "/lib/x86_64-linux-gnu/libthread_db.so.1".
W20260810 22:43:28.799551 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1791: this message will repeat every 5th retry.
W20260810 22:43:30.027596 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1786: this message will repeat every 5th retry.
W20260810 22:43:30.598774 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1781: this message will repeat every 5th retry.
W20260810 22:43:30.821954 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 29 similar messages]
W20260810 22:43:31.274026 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1796: this message will repeat every 5th retry.
W20260810 22:43:32.247114 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1791: this message will repeat every 5th retry.
0x00007f168184cad3 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
Id Target Id Frame
* 1 Thread 0x7f167be9a240 (LWP 8266) "all_types-itest" 0x00007f168184cad3 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
2 Thread 0x7f167b63c700 (LWP 8271) "all_types-itest" 0x00007f168185132a in waitpid () from /lib/x86_64-linux-gnu/libpthread.so.0
3 Thread 0x7f167a073700 (LWP 8272) "kernel-watcher-" 0x00007f168184cfb9 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
4 Thread 0x7f1679872700 (LWP 8274) "rpc reactor-827" 0x00007f167cc6b947 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
5 Thread 0x7f1679071700 (LWP 8936) "rpc reactor-893" 0x00007f167cc6b947 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
6 Thread 0x7f1678669700 (LWP 8937) "rpc reactor-893" 0x00007f167cc6b947 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
7 Thread 0x7f1677e68700 (LWP 8938) "rpc reactor-893" 0x00007f167cc6b947 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
8 Thread 0x7f1677667700 (LWP 8940) "rpc reactor-894" 0x00007f167cc6b947 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
9 Thread 0x7f1674661700 (LWP 14525) "rpc reactor-145" 0x00007f167cc6b947 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
10 Thread 0x7f1670659700 (LWP 14526) "rpc reactor-145" 0x00007f167cc6b947 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
11 Thread 0x7f167265d700 (LWP 14527) "rpc reactor-145" 0x00007f167cc6b947 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
12 Thread 0x7f1670e5a700 (LWP 14528) "rpc reactor-145" 0x00007f167cc6b947 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
13 Thread 0x7f1675e64700 (LWP 14534) "rpc reactor-145" 0x00007f167cc6b947 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
14 Thread 0x7f1675663700 (LWP 14535) "rpc reactor-145" 0x00007f167cc6b947 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
15 Thread 0x7f1674e62700 (LWP 14536) "rpc reactor-145" 0x00007f167cc6b947 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
16 Thread 0x7f1673e60700 (LWP 14537) "rpc reactor-145" 0x00007f167cc6b947 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
17 Thread 0x7f167365f700 (LWP 14539) "client-negotiat" 0x00007f168184cfb9 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
Thread 17 (Thread 0x7f167365f700 (LWP 14539)):
#0 0x00007f168184cfb9 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
#1 0x00007f16805f1de9 in kudu::ConditionVariable::WaitFor (this=0x7f167365e140, delta=...) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/condition_variable.cc:121
#2 0x00007f1680726ecf in kudu::ThreadPool::DispatchThread (this=0x3f7caf00) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/threadpool.cc:721
#3 0x00007f1680727c83 in kudu::ThreadPool::<lambda()>::operator()(void) const (__closure=0x3fcff138) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/threadpool.cc:849
W20260810 22:43:33.111845 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1786: this message will repeat every 5th retry.
#4 0x00007f16807293f2 in std::_Function_handler<void(), kudu::ThreadPool::CreateThread()::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...) at /opt/rh/devtoolset-8/root/usr/include/c++/8/bits/std_function.h:297
#5 0x00007f1689560170 in std::function<void ()>::operator()() const (this=0x3fcff138) at /opt/rh/devtoolset-8/root/usr/include/c++/8/bits/std_function.h:687
#6 0x00007f1680718512 in kudu::Thread::SuperviseThread (arg=0x3fcff110) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/thread.cc:694
#7 0x00007f16818466db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#8 0x00007f167cc6b61f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 16 (Thread 0x7f1673e60700 (LWP 14537)):
#0 0x00007f167cc6b947 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x00007f167f6a6bda in epoll_poll (loop=0x3f5f7200, timeout=0.099045129958540201) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/libev-4.20/ev_epoll.c:153
#2 0x00007f167f6aa228 in ev_run (loop=0x3f5f7200, flags=0) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/libev-4.20/ev.c:3521
#3 0x00007f1684964207 in ev::loop_ref::run (this=0x3fe2ea08, flags=0) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/uninstrumented/include/ev++.h:211
#4 0x00007f168495f34a in kudu::rpc::ReactorThread::RunThread (this=0x3fe2ea00) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/reactor.cc:510
#5 0x00007f168495bfb5 in kudu::rpc::ReactorThread::<lambda()>::operator()(void) const (__closure=0x3fcff618) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/reactor.cc:197
#6 0x00007f168496344d in std::_Function_handler<void(), kudu::rpc::ReactorThread::Init()::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...) at /opt/rh/devtoolset-8/root/usr/include/c++/8/bits/std_function.h:297
#7 0x00007f1689560170 in std::function<void ()>::operator()() const (this=0x3fcff618) at /opt/rh/devtoolset-8/root/usr/include/c++/8/bits/std_function.h:687
#8 0x00007f1680718512 in kudu::Thread::SuperviseThread (arg=0x3fcff5f0) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/thread.cc:694
#9 0x00007f16818466db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#10 0x00007f167cc6b61f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 15 (Thread 0x7f1674e62700 (LWP 14536)):
#0 0x00007f167cc6b947 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x00007f167f6a6bda in epoll_poll (loop=0x3f59fb00, timeout=0.099468264961615205) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/libev-4.20/ev_epoll.c:153
#2 0x00007f167f6aa228 in ev_run (loop=0x3f59fb00, flags=0) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/libev-4.20/ev.c:3521
#3 0x00007f1684964207 in ev::loop_ref::run (this=0x3fe2e868, flags=0) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/uninstrumented/include/ev++.h:211
#4 0x00007f168495f34a in kudu::rpc::ReactorThread::RunThread (this=0x3fe2e860) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/reactor.cc:510
#5 0x00007f168495bfb5 in kudu::rpc::ReactorThread::<lambda()>::operator()(void) const (__closure=0x3fcfe778) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/reactor.cc:197
#6 0x00007f168496344d in std::_Function_handler<void(), kudu::rpc::ReactorThread::Init()::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...) at /opt/rh/devtoolset-8/root/usr/include/c++/8/bits/std_function.h:297
#7 0x00007f1689560170 in std::function<void ()>::operator()() const (this=0x3fcfe778) at /opt/rh/devtoolset-8/root/usr/include/c++/8/bits/std_function.h:687
#8 0x00007f1680718512 in kudu::Thread::SuperviseThread (arg=0x3fcfe750) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/thread.cc:694
#9 0x00007f16818466db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#10 0x00007f167cc6b61f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 14 (Thread 0x7f1675663700 (LWP 14535)):
#0 0x00007f167cc6b947 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x00007f167f6a6bda in epoll_poll (loop=0x3f5a1600, timeout=0.099031712859869003) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/libev-4.20/ev_epoll.c:153
#2 0x00007f167f6aa228 in ev_run (loop=0x3f5a1600, flags=0) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/libev-4.20/ev.c:3521
#3 0x00007f1684964207 in ev::loop_ref::run (this=0x3fe2e6c8, flags=0) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/uninstrumented/include/ev++.h:211
#4 0x00007f168495f34a in kudu::rpc::ReactorThread::RunThread (this=0x3fe2e6c0) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/reactor.cc:510
#5 0x00007f168495bfb5 in kudu::rpc::ReactorThread::<lambda()>::operator()(void) const (__closure=0x3fcfe5d8) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/reactor.cc:197
#6 0x00007f168496344d in std::_Function_handler<void(), kudu::rpc::ReactorThread::Init()::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...) at /opt/rh/devtoolset-8/root/usr/include/c++/8/bits/std_function.h:297
#7 0x00007f1689560170 in std::function<void ()>::operator()() const (this=0x3fcfe5d8) at /opt/rh/devtoolset-8/root/usr/include/c++/8/bits/std_function.h:687
#8 0x00007f1680718512 in kudu::Thread::SuperviseThread (arg=0x3fcfe5b0) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/thread.cc:694
#9 0x00007f16818466db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#10 0x00007f167cc6b61f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 13 (Thread 0x7f1675e64700 (LWP 14534)):
#0 0x00007f167cc6b947 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x00007f167f6a6bda in epoll_poll (loop=0x3f59ff80, timeout=0.084938758984208107) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/libev-4.20/ev_epoll.c:153
#2 0x00007f167f6aa228 in ev_run (loop=0x3f59ff80, flags=0) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/libev-4.20/ev.c:3521
#3 0x00007f1684964207 in ev::loop_ref::run (this=0x3fe2e528, flags=0) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/uninstrumented/include/ev++.h:211
#4 0x00007f168495f34a in kudu::rpc::ReactorThread::RunThread (this=0x3fe2e520) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/reactor.cc:510
#5 0x00007f168495bfb5 in kudu::rpc::ReactorThread::<lambda()>::operator()(void) const (__closure=0x3fcff3a8) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/reactor.cc:197
#6 0x00007f168496344d in std::_Function_handler<void(), kudu::rpc::ReactorThread::Init()::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...) at /opt/rh/devtoolset-8/root/usr/include/c++/8/bits/std_function.h:297
#7 0x00007f1689560170 in std::function<void ()>::operator()() const (this=0x3fcff3a8) at /opt/rh/devtoolset-8/root/usr/include/c++/8/bits/std_function.h:687
#8 0x00007f1680718512 in kudu::Thread::SuperviseThread (arg=0x3fcff380) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/thread.cc:694
#9 0x00007f16818466db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#10 0x00007f167cc6b61f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 12 (Thread 0x7f1670e5a700 (LWP 14528)):
#0 0x00007f167cc6b947 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x00007f167f6a6bda in epoll_poll (loop=0x3f5a0d00, timeout=0.099070022813975811) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/libev-4.20/ev_epoll.c:153
#2 0x00007f167f6aa228 in ev_run (loop=0x3f5a0d00, flags=0) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/libev-4.20/ev.c:3521
#3 0x00007f1684964207 in ev::loop_ref::run (this=0x3f52dd88, flags=0) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/uninstrumented/include/ev++.h:211
#4 0x00007f168495f34a in kudu::rpc::ReactorThread::RunThread (this=0x3f52dd80) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/reactor.cc:510
#5 0x00007f168495bfb5 in kudu::rpc::ReactorThread::<lambda()>::operator()(void) const (__closure=0x3f55fe38) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/reactor.cc:197
#6 0x00007f168496344d in std::_Function_handler<void(), kudu::rpc::ReactorThread::Init()::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...) at /opt/rh/devtoolset-8/root/usr/include/c++/8/bits/std_function.h:297
#7 0x00007f1689560170 in std::function<void ()>::operator()() const (this=0x3f55fe38) at /opt/rh/devtoolset-8/root/usr/include/c++/8/bits/std_function.h:687
#8 0x00007f1680718512 in kudu::Thread::SuperviseThread (arg=0x3f55fe10) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/thread.cc:694
#9 0x00007f16818466db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#10 0x00007f167cc6b61f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 11 (Thread 0x7f167265d700 (LWP 14527)):
#0 0x00007f167cc6b947 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x00007f167f6a6bda in epoll_poll (loop=0x3f5a1180, timeout=0.099048698088154197) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/libev-4.20/ev_epoll.c:153
#2 0x00007f167f6aa228 in ev_run (loop=0x3f5a1180, flags=0) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/libev-4.20/ev.c:3521
#3 0x00007f1684964207 in ev::loop_ref::run (this=0x3f52d228, flags=0) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/uninstrumented/include/ev++.h:211
#4 0x00007f168495f34a in kudu::rpc::ReactorThread::RunThread (this=0x3f52d220) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/reactor.cc:510
#5 0x00007f168495bfb5 in kudu::rpc::ReactorThread::<lambda()>::operator()(void) const (__closure=0x3f55ef98) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/reactor.cc:197
#6 0x00007f168496344d in std::_Function_handler<void(), kudu::rpc::ReactorThread::Init()::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...) at /opt/rh/devtoolset-8/root/usr/include/c++/8/bits/std_function.h:297
#7 0x00007f1689560170 in std::function<void ()>::operator()() const (this=0x3f55ef98) at /opt/rh/devtoolset-8/root/usr/include/c++/8/bits/std_function.h:687
#8 0x00007f1680718512 in kudu::Thread::SuperviseThread (arg=0x3f55ef70) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/thread.cc:694
#9 0x00007f16818466db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#10 0x00007f167cc6b61f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 10 (Thread 0x7f1670659700 (LWP 14526)):
#0 0x00007f167cc6b947 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x00007f167f6a6bda in epoll_poll (loop=0x3f5a1a80, timeout=0.099028132855892181) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/libev-4.20/ev_epoll.c:153
#2 0x00007f167f6aa228 in ev_run (loop=0x3f5a1a80, flags=0) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/libev-4.20/ev.c:3521
#3 0x00007f1684964207 in ev::loop_ref::run (this=0x3f52dbe8, flags=0) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/uninstrumented/include/ev++.h:211
#4 0x00007f168495f34a in kudu::rpc::ReactorThread::RunThread (this=0x3f52dbe0) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/reactor.cc:510
#5 0x00007f168495bfb5 in kudu::rpc::ReactorThread::<lambda()>::operator()(void) const (__closure=0x3f55ec58) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/reactor.cc:197
#6 0x00007f168496344d in std::_Function_handler<void(), kudu::rpc::ReactorThread::Init()::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...) at /opt/rh/devtoolset-8/root/usr/include/c++/8/bits/std_function.h:297
#7 0x00007f1689560170 in std::function<void ()>::operator()() const (this=0x3f55ec58) at /opt/rh/devtoolset-8/root/usr/include/c++/8/bits/std_function.h:687
#8 0x00007f1680718512 in kudu::Thread::SuperviseThread (arg=0x3f55ec30) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/thread.cc:694
#9 0x00007f16818466db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#10 0x00007f167cc6b61f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 9 (Thread 0x7f1674661700 (LWP 14525)):
#0 0x00007f167cc6b947 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x00007f167f6a6bda in epoll_poll (loop=0x3f5f8400, timeout=0.099684660090133548) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/libev-4.20/ev_epoll.c:153
#2 0x00007f167f6aa228 in ev_run (loop=0x3f5f8400, flags=0) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/libev-4.20/ev.c:3521
#3 0x00007f1684964207 in ev::loop_ref::run (this=0x3f52da48, flags=0) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/uninstrumented/include/ev++.h:211
#4 0x00007f168495f34a in kudu::rpc::ReactorThread::RunThread (this=0x3f52da40) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/reactor.cc:510
#5 0x00007f168495bfb5 in kudu::rpc::ReactorThread::<lambda()>::operator()(void) const (__closure=0x3f55eb88) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/reactor.cc:197
#6 0x00007f168496344d in std::_Function_handler<void(), kudu::rpc::ReactorThread::Init()::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...) at /opt/rh/devtoolset-8/root/usr/include/c++/8/bits/std_function.h:297
#7 0x00007f1689560170 in std::function<void ()>::operator()() const (this=0x3f55eb88) at /opt/rh/devtoolset-8/root/usr/include/c++/8/bits/std_function.h:687
#8 0x00007f1680718512 in kudu::Thread::SuperviseThread (arg=0x3f55eb60) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/thread.cc:694
#9 0x00007f16818466db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#10 0x00007f167cc6b61f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 8 (Thread 0x7f1677667700 (LWP 8940)):
#0 0x00007f167cc6b947 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x00007f167f6a6bda in epoll_poll (loop=0x3f59f680, timeout=0.099378332030028105) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/libev-4.20/ev_epoll.c:153
#2 0x00007f167f6aa228 in ev_run (loop=0x3f59f680, flags=0) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/libev-4.20/ev.c:3521
#3 0x00007f1684964207 in ev::loop_ref::run (this=0x3f52d088, flags=0) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/uninstrumented/include/ev++.h:211
#4 0x00007f168495f34a in kudu::rpc::ReactorThread::RunThread (this=0x3f52d080) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/reactor.cc:510
#5 0x00007f168495bfb5 in kudu::rpc::ReactorThread::<lambda()>::operator()(void) const (__closure=0x3f55f6e8) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/reactor.cc:197
#6 0x00007f168496344d in std::_Function_handler<void(), kudu::rpc::ReactorThread::Init()::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...) at /opt/rh/devtoolset-8/root/usr/include/c++/8/bits/std_function.h:297
#7 0x00007f1689560170 in std::function<void ()>::operator()() const (this=0x3f55f6e8) at /opt/rh/devtoolset-8/root/usr/include/c++/8/bits/std_function.h:687
#8 0x00007f1680718512 in kudu::Thread::SuperviseThread (arg=0x3f55f6c0) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/thread.cc:694
#9 0x00007f16818466db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#10 0x00007f167cc6b61f in clone () from /lib/x86_64-linux-gnu/libc.so.6
W20260810 22:43:34.062345 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1801: this message will repeat every 5th retry.
Thread 7 (Thread 0x7f1677e68700 (LWP 8938)):
#0 0x00007f167cc6b947 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x00007f167f6a6bda in epoll_poll (loop=0x3f59f200, timeout=0.098935908870771527) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/libev-4.20/ev_epoll.c:153
#2 0x00007f167f6aa228 in ev_run (loop=0x3f59f200, flags=0) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/libev-4.20/ev.c:3521
#3 0x00007f1684964207 in ev::loop_ref::run (this=0x3f52cee8, flags=0) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/uninstrumented/include/ev++.h:211
#4 0x00007f168495f34a in kudu::rpc::ReactorThread::RunThread (this=0x3f52cee0) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/reactor.cc:510
#5 0x00007f168495bfb5 in kudu::rpc::ReactorThread::<lambda()>::operator()(void) const (__closure=0x3f55f618) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/reactor.cc:197
#6 0x00007f168496344d in std::_Function_handler<void(), kudu::rpc::ReactorThread::Init()::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...) at /opt/rh/devtoolset-8/root/usr/include/c++/8/bits/std_function.h:297
#7 0x00007f1689560170 in std::function<void ()>::operator()() const (this=0x3f55f618) at /opt/rh/devtoolset-8/root/usr/include/c++/8/bits/std_function.h:687
#8 0x00007f1680718512 in kudu::Thread::SuperviseThread (arg=0x3f55f5f0) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/thread.cc:694
#9 0x00007f16818466db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#10 0x00007f167cc6b61f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 6 (Thread 0x7f1678669700 (LWP 8937)):
#0 0x00007f167cc6b947 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x00007f167f6a6bda in epoll_poll (loop=0x3f59ed80, timeout=0.099369864445179701) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/libev-4.20/ev_epoll.c:153
#2 0x00007f167f6aa228 in ev_run (loop=0x3f59ed80, flags=0) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/libev-4.20/ev.c:3521
#3 0x00007f1684964207 in ev::loop_ref::run (this=0x3f52ca08, flags=0) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/uninstrumented/include/ev++.h:211
#4 0x00007f168495f34a in kudu::rpc::ReactorThread::RunThread (this=0x3f52ca00) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/reactor.cc:510
#5 0x00007f168495bfb5 in kudu::rpc::ReactorThread::<lambda()>::operator()(void) const (__closure=0x3f55f548) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/reactor.cc:197
#6 0x00007f168496344d in std::_Function_handler<void(), kudu::rpc::ReactorThread::Init()::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...) at /opt/rh/devtoolset-8/root/usr/include/c++/8/bits/std_function.h:297
#7 0x00007f1689560170 in std::function<void ()>::operator()() const (this=0x3f55f548) at /opt/rh/devtoolset-8/root/usr/include/c++/8/bits/std_function.h:687
#8 0x00007f1680718512 in kudu::Thread::SuperviseThread (arg=0x3f55f520) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/thread.cc:694
#9 0x00007f16818466db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#10 0x00007f167cc6b61f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 5 (Thread 0x7f1679071700 (LWP 8936)):
#0 0x00007f167cc6b947 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x00007f167f6a6bda in epoll_poll (loop=0x3f59e900, timeout=0.09997618873603642) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/libev-4.20/ev_epoll.c:153
#2 0x00007f167f6aa228 in ev_run (loop=0x3f59e900, flags=0) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/libev-4.20/ev.c:3521
#3 0x00007f1684964207 in ev::loop_ref::run (this=0x3f52cd48, flags=0) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/uninstrumented/include/ev++.h:211
#4 0x00007f168495f34a in kudu::rpc::ReactorThread::RunThread (this=0x3f52cd40) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/reactor.cc:510
#5 0x00007f168495bfb5 in kudu::rpc::ReactorThread::<lambda()>::operator()(void) const (__closure=0x3f55f478) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/reactor.cc:197
#6 0x00007f168496344d in std::_Function_handler<void(), kudu::rpc::ReactorThread::Init()::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...) at /opt/rh/devtoolset-8/root/usr/include/c++/8/bits/std_function.h:297
#7 0x00007f1689560170 in std::function<void ()>::operator()() const (this=0x3f55f478) at /opt/rh/devtoolset-8/root/usr/include/c++/8/bits/std_function.h:687
#8 0x00007f1680718512 in kudu::Thread::SuperviseThread (arg=0x3f55f450) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/thread.cc:694
#9 0x00007f16818466db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#10 0x00007f167cc6b61f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 4 (Thread 0x7f1679872700 (LWP 8274)):
#0 0x00007f167cc6b947 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x00007f167f6a6bda in epoll_poll (loop=0x3f59e480, timeout=0.099927821662276983) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/libev-4.20/ev_epoll.c:153
#2 0x00007f167f6aa228 in ev_run (loop=0x3f59e480, flags=0) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/libev-4.20/ev.c:3521
#3 0x00007f1684964207 in ev::loop_ref::run (this=0x3f52c388, flags=0) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/uninstrumented/include/ev++.h:211
#4 0x00007f168495f34a in kudu::rpc::ReactorThread::RunThread (this=0x3f52c380) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/reactor.cc:510
#5 0x00007f168495bfb5 in kudu::rpc::ReactorThread::<lambda()>::operator()(void) const (__closure=0x3f55e368) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/reactor.cc:197
#6 0x00007f168496344d in std::_Function_handler<void(), kudu::rpc::ReactorThread::Init()::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...) at /opt/rh/devtoolset-8/root/usr/include/c++/8/bits/std_function.h:297
#7 0x00007f1689560170 in std::function<void ()>::operator()() const (this=0x3f55e368) at /opt/rh/devtoolset-8/root/usr/include/c++/8/bits/std_function.h:687
#8 0x00007f1680718512 in kudu::Thread::SuperviseThread (arg=0x3f55e340) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/thread.cc:694
#9 0x00007f16818466db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#10 0x00007f167cc6b61f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 3 (Thread 0x7f167a073700 (LWP 8272)):
#0 0x00007f168184cfb9 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
W20260810 22:43:34.447619 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1796: this message will repeat every 5th retry.
#1 0x00007f16805f1c38 in kudu::ConditionVariable::WaitUntil (this=0x3f411878, until=...) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/condition_variable.cc:87
#2 0x00007f16891a94e1 in kudu::CountDownLatch::WaitUntil (this=0x3f411840, when=...) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/countdown_latch.h:89
#3 0x00007f16891a9569 in kudu::CountDownLatch::WaitFor (this=0x3f411840, delta=...) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/countdown_latch.h:99
#4 0x00007f168067df57 in kudu::KernelStackWatchdog::RunThread (this=0x3f4117a0) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/kernel_stack_watchdog.cc:134
#5 0x00007f168067d291 in kudu::KernelStackWatchdog::<lambda()>::operator()(void) const (__closure=0x3f460ec8) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/kernel_stack_watchdog.cc:71
#6 0x00007f168067e7f3 in std::_Function_handler<void(), kudu::KernelStackWatchdog::KernelStackWatchdog()::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...) at /opt/rh/devtoolset-8/root/usr/include/c++/8/bits/std_function.h:297
#7 0x00007f1689560170 in std::function<void ()>::operator()() const (this=0x3f460ec8) at /opt/rh/devtoolset-8/root/usr/include/c++/8/bits/std_function.h:687
#8 0x00007f1680718512 in kudu::Thread::SuperviseThread (arg=0x3f460ea0) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/thread.cc:694
#9 0x00007f16818466db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#10 0x00007f167cc6b61f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 2 (Thread 0x7f167b63c700 (LWP 8271)):
#0 0x00007f168185132a in waitpid () from /lib/x86_64-linux-gnu/libpthread.so.0
#1 0x00007f168070fe73 in kudu::Subprocess::DoWait (this=0x7f167b63b2a0, wait_status=0x0, mode=kudu::Subprocess::BLOCKING) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/subprocess.cc:817
#2 0x00007f168070d433 in kudu::Subprocess::Wait (this=0x7f167b63b2a0, wait_status=0x0) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/subprocess.cc:544
#3 0x00007f168066c869 in kudu::PstackWatcher::RunStackDump (argv=std::vector of length 12, capacity 16 = {...}) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/pstack_watcher.cc:258
#4 0x00007f168066c418 in kudu::PstackWatcher::RunGdbStackDump (pid=8266, flags=0) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/pstack_watcher.cc:218
#5 0x00007f168066bd1a in kudu::PstackWatcher::DumpPidStacks (pid=8266, flags=0) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/pstack_watcher.cc:169
#6 0x00007f168066bcb5 in kudu::PstackWatcher::DumpStacks (flags=0) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/pstack_watcher.cc:162
#7 0x0000000000513498 in kudu::<lambda()>::operator()(void) const (__closure=0x3f439bc8) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/test_main.cc:60
#8 0x0000000000513bae in std::__invoke_impl<void, kudu::CreateAndStartTimeoutThread()::<lambda()> >(std::__invoke_other, kudu::<lambda()> &&) (__f=...) at /opt/rh/devtoolset-8/root/usr/include/c++/8/bits/invoke.h:60
#9 0x0000000000513960 in std::__invoke<kudu::CreateAndStartTimeoutThread()::<lambda()> >(kudu::<lambda()> &&) (__fn=...) at /opt/rh/devtoolset-8/root/usr/include/c++/8/bits/invoke.h:95
#10 0x0000000000514180 in std::thread::_Invoker<std::tuple<kudu::CreateAndStartTimeoutThread()::<lambda()> > >::_M_invoke<0>(std::_Index_tuple<0>) (this=0x3f439bc8) at /opt/rh/devtoolset-8/root/usr/include/c++/8/thread:244
#11 0x000000000051412a in std::thread::_Invoker<std::tuple<kudu::CreateAndStartTimeoutThread()::<lambda()> > >::operator()(void) (this=0x3f439bc8) at /opt/rh/devtoolset-8/root/usr/include/c++/8/thread:253
#12 0x00000000005140f4 in std::thread::_State_impl<std::thread::_Invoker<std::tuple<kudu::CreateAndStartTimeoutThread()::<lambda()> > > >::_M_run(void) (this=0x3f439bc0) at /opt/rh/devtoolset-8/root/usr/include/c++/8/thread:196
#13 0x00007f168555fa4f in execute_native_thread_routine () from /tmp/dist-test-taskMIzrRq/build/debug/bin/../../../thirdparty/installed/uninstrumented/lib/librocksdb.so.7
#14 0x00007f16818466db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#15 0x00007f167cc6b61f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 1 (Thread 0x7f167be9a240 (LWP 8266)):
#0 0x00007f168184cad3 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
#1 0x00007f16805f1ade in kudu::ConditionVariable::Wait (this=0x3f9293b0) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/condition_variable.cc:55
#2 0x00007f1689977114 in kudu::CountDownLatch::Wait (this=0x3f929378) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/countdown_latch.h:79
W20260810 22:43:35.551724 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1791: this message will repeat every 5th retry.
#3 0x00007f168949552a in kudu::Synchronizer::Wait (this=0x7ffd54662880) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/async_util.h:58
W20260810 22:43:35.855316 8609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111) [suppressed 30 similar messages]
#4 0x00007f1687562d7d in kudu::client::KuduClient::Data::ConnectToCluster (this=0x3f684b40, client=0x3fe3df60, deadline=..., creds_policy=kudu::rpc::CredentialsPolicy::ANY_CREDENTIALS) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/client/client-internal.cc:815
W20260810 22:43:36.490396 8579 heartbeater.cc:644] Failed to heartbeat to 127.8.18.190:40323 (780 consecutive failures): Network error: Failed to ping master at 127.8.18.190:40323: Client connection negotiation failed: client connection to 127.8.18.190:40323: connect: Connection refused (error 111)
W20260810 22:43:36.624752 8756 heartbeater.cc:644] Failed to heartbeat to 127.8.18.190:40323 (782 consecutive failures): Network error: Failed to ping master at 127.8.18.190:40323: Client connection negotiation failed: client connection to 127.8.18.190:40323: connect: Connection refused (error 111)
W20260810 22:43:36.791934 8609 consensus_peers.cc:487] T 459e1d55c65e4857a7ccba6b13316243 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1806: this message will repeat every 5th retry.
#5 0x00007f16875280e4 in kudu::client::KuduClientBuilder::Build (this=0x7ffd54662c38, client=0x3fd4f310) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/client/client.cc:469
W20260810 22:43:36.958853 8609 consensus_peers.cc:487] T b29f930721b943039d1df1f03e073a74 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1801: this message will repeat every 5th retry.
#6 0x00007f1687847bd1 in kudu::tools::CreateKuduClient (master_addresses=std::vector of length 1, capacity 1 = {...}, client=0x3fd4f310, can_see_all_replicas=true) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/tools/tool_action_common.cc:738
#7 0x00007f16891bfc9d in kudu::tools::RemoteKsckCluster::Connect (this=0x3fd4f200) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/tools/ksck_remote.cc:530
#8 0x00007f168917d33f in kudu::tools::Ksck::CheckClusterRunning (this=0x3f5ed810) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/tools/ksck.cc:460
#9 0x00007f168917e80a in kudu::tools::Ksck::Run (this=0x3f5ed810) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/tools/ksck.cc:595
#10 0x00007f1689180a86 in kudu::tools::Ksck::RunAndPrintResults (this=0x3f5ed810) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/tools/ksck.cc:801
#11 0x00007f1689949640 in kudu::ClusterVerifier::RunKsck (this=0x7ffd54663410) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/integration-tests/cluster_verifier.cc:113
W20260810 22:43:38.219985 8609 consensus_peers.cc:487] T 1f99182d93ed483e9f520bf312a591e3 P 01cff8a283e94658aea5fad9fa0b2db4 -> Peer d06d8f442c824758ae944ffe71e3cb44 (127.8.18.131:36507): Couldn't send request to peer d06d8f442c824758ae944ffe71e3cb44. Status: Network error: Client connection negotiation failed: client connection to 127.8.18.131:36507: connect: Connection refused (error 111). This is attempt 1796: this message will repeat every 5th retry.
#12 0x00007f16899490a5 in kudu::ClusterVerifier::CheckCluster (this=0x7ffd54663410) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/integration-tests/cluster_verifier.cc:78
#13 0x00000000004d06d5 in kudu::client::AllTypesItest<kudu::client::IntKeysTestSetup<kudu::client::KeyTypeWrapper<(kudu::DataType)3> > >::RunTest(std::function<kudu::Status (kudu::client::KuduScanner*)> const&, std::function<void (kudu::client::KuduScanBatch const&, int, int*)> const&) (this=0x3f52e900, scanner_setup=..., verifier=...) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/integration-tests/all_types-itest.cc:622
#14 0x00000000004a451c in kudu::client::AllTypesItest_TestTimestampPadding_Test<kudu::client::IntKeysTestSetup<kudu::client::KeyTypeWrapper<(kudu::DataType)3> > >::TestBody (this=0x3f52e900) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/integration-tests/all_types-itest.cc:804
#15 0x00007f16809a9bfe in testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test, void> (location=0x7f16809afa28 "the test body", method=<optimized out>, object=0x3f52e900) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest-internal-inl.h:831
#16 testing::internal::HandleExceptionsInMethodIfSupported<testing::Test, void> (object=object@entry=0x3f52e900, method=<optimized out>, location=location@entry=0x7f16809afa28 "the test body") at /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635
#17 0x00007f168099e5db in testing::Test::Run (this=0x3f52e900) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2674
#18 testing::Test::Run (this=0x3f52e900) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2664
#19 0x00007f168099e795 in testing::TestInfo::Run (this=0x3f460340) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2853
#20 0x00007f168099ecdf in testing::TestSuite::Run (this=0x3f510000) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:3012
#21 testing::TestSuite::Run (this=0x3f510000) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2985
#22 0x00007f168099f29f in testing::internal::UnitTestImpl::RunAllTests() () at /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest-internal-inl.h:600
#23 0x00007f16809aa13e in testing::internal::HandleSehExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool> (location=0x7f16809b15a8 "auxiliary test code (environments or event listeners)", method=<optimized out>, object=0x3f44c280) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest-internal-inl.h:831
#24 testing::internal::HandleExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool> (object=0x3f44c280, method=<optimized out>, location=location@entry=0x7f16809b15a8 "auxiliary test code (environments or event listeners)") at /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635
#25 0x00007f168099e83d in testing::UnitTest::Run() () at /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/googletest-release-1.12.1/googletest/include/gtest/gtest.h:1246
#26 0x00000000005144b0 in RUN_ALL_TESTS () at /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/uninstrumented/include/gtest/gtest.h:2293
#27 0x00000000005138da in main (argc=1, argv=0x7ffd54663978) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/test_main.cc:109
************************* END STACKS ***************************
F20260810 22:43:38.902309 8271 test_main.cc:63] Maximum unit test time exceeded (870 sec)
*** Check failure stack trace: ***
*** Aborted at 1786401818 (unix time) try "date -d @1786401818" if you are using GNU date ***
PC: @ 0x0 (unknown)
*** SIGABRT (@0x3e80000204a) received by PID 8266 (TID 0x7f167b63c700) from PID 8266; stack trace: ***
@ 0x7f167f0d3a46 google::(anonymous namespace)::FailureSignalHandler() at ??:0
@ 0x7f1681851980 (unknown) at ??:0
@ 0x7f167cb88e87 gsignal at ??:0
@ 0x7f167cb8a7f1 abort at ??:0
@ 0x7f167f0c59ad google::LogMessage::Fail() at ??:0
@ 0x7f167f0c888a google::LogMessage::SendToLog() at ??:0
@ 0x7f167f0c54de google::LogMessage::Flush() at ??:0
@ 0x7f167f0c6a19 google::LogMessageFatal::~LogMessageFatal() at ??:0
@ 0x5135cc _ZZN4kuduL27CreateAndStartTimeoutThreadEvENKUlvE_clEv at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/test_main.cc:63 (discriminator 9)
@ 0x513bae _ZSt13__invoke_implIvZN4kuduL27CreateAndStartTimeoutThreadEvEUlvE_JEET_St14__invoke_otherOT0_DpOT1_ at /opt/rh/devtoolset-8/root/usr/include/c++/8/bits/invoke.h:60
@ 0x513960 _ZSt8__invokeIZN4kuduL27CreateAndStartTimeoutThreadEvEUlvE_JEENSt15__invoke_resultIT_JDpT0_EE4typeEOS3_DpOS4_ at /opt/rh/devtoolset-8/root/usr/include/c++/8/bits/invoke.h:96
@ 0x514180 _ZNSt6thread8_InvokerISt5tupleIJZN4kuduL27CreateAndStartTimeoutThreadEvEUlvE_EEE9_M_invokeIJLm0EEEEDTcl8__invokespcl10_S_declvalIXT_EEEEESt12_Index_tupleIJXspT_EEE at /opt/rh/devtoolset-8/root/usr/include/c++/8/thread:244
@ 0x51412a _ZNSt6thread8_InvokerISt5tupleIJZN4kuduL27CreateAndStartTimeoutThreadEvEUlvE_EEEclEv at /opt/rh/devtoolset-8/root/usr/include/c++/8/thread:253
@ 0x5140f4 _ZNSt6thread11_State_implINS_8_InvokerISt5tupleIJZN4kuduL27CreateAndStartTimeoutThreadEvEUlvE_EEEEE6_M_runEv at /opt/rh/devtoolset-8/root/usr/include/c++/8/thread:196
@ 0x7f168555fa4f execute_native_thread_routine at ??:0
@ 0x7f16818466db start_thread at ??:0
@ 0x7f167cc6b61f clone at ??:0